diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 230c2a023..9a107f88b 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -45,6 +45,7 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, addSharePointCommands, addGroupsCommands, + addTeamsChatsCommands, } // AddCommands attaches all `corso backup * *` commands to the parent. diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go index 2a8ab3024..fad35ce5d 100644 --- a/src/cli/backup/groups.go +++ b/src/cli/backup/groups.go @@ -310,7 +310,7 @@ func groupsBackupCreateSelectors( group, cats []string, ) *selectors.GroupsBackup { if filters.PathContains(group).Compare(flags.Wildcard) { - return includeAllGroupWithCategories(ins, cats) + return includeAllGroupsWithCategories(ins, cats) } sel := selectors.NewGroupsBackup(slices.Clone(group)) @@ -318,6 +318,6 @@ func groupsBackupCreateSelectors( return utils.AddGroupsCategories(sel, cats) } -func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup { +func includeAllGroupsWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup { return utils.AddGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories) } diff --git a/src/cli/backup/teamschats.go b/src/cli/backup/teamschats.go new file mode 100644 index 000000000..190189380 --- /dev/null +++ b/src/cli/backup/teamschats.go @@ -0,0 +1,305 @@ +package backup + +import ( + "context" + "fmt" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365" +) + +// ------------------------------------------------------------------------------------------------ +// setup and globals +// ------------------------------------------------------------------------------------------------ + +const ( + teamschatsServiceCommand = "chats" + teamschatsServiceCommandCreateUseSuffix = "--user | '" + flags.Wildcard + "'" + teamschatsServiceCommandDeleteUseSuffix = "--backups " + teamschatsServiceCommandDetailsUseSuffix = "--backup " +) + +const ( + teamschatsServiceCommandCreateExamples = `# Backup all chats with bob@company.hr +corso backup create chats --user bob@company.hr + +# Backup all chats for all users +corso backup create chats --user '*'` + + teamschatsServiceCommandDeleteExamples = `# Delete chats backup with ID 1234abcd-12ab-cd34-56de-1234abcd \ +and 1234abcd-12ab-cd34-56de-1234abce +corso backup delete chats --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce` + + teamschatsServiceCommandDetailsExamples = `# Explore chats in Bob's latest backup (1234abcd...) +corso backup details chats --backup 1234abcd-12ab-cd34-56de-1234abcd` +) + +// called by backup.go to map subcommands to provider-specific handling. +func addTeamsChatsCommands(cmd *cobra.Command) *cobra.Command { + var c *cobra.Command + + switch cmd.Use { + case createCommand: + c, _ = utils.AddCommand(cmd, teamschatsCreateCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandCreateUseSuffix + c.Example = teamschatsServiceCommandCreateExamples + + // Flags addition ordering should follow the order we want them to appear in help and docs: + flags.AddUserFlag(c) + flags.AddDataFlag(c, []string{flags.DataChats}, false) + flags.AddGenericBackupFlags(c) + + case listCommand: + c, _ = utils.AddCommand(cmd, teamschatsListCmd(), utils.MarkPreReleaseCommand()) + + flags.AddBackupIDFlag(c, false) + flags.AddAllBackupListFlags(c) + + case detailsCommand: + c, _ = utils.AddCommand(cmd, teamschatsDetailsCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandDetailsUseSuffix + c.Example = teamschatsServiceCommandDetailsExamples + + flags.AddSkipReduceFlag(c) + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + flags.AddBackupIDFlag(c, true) + flags.AddTeamsChatsDetailsAndRestoreFlags(c) + + case deleteCommand: + c, _ = utils.AddCommand(cmd, teamschatsDeleteCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandDeleteUseSuffix + c.Example = teamschatsServiceCommandDeleteExamples + + flags.AddMultipleBackupIDsFlag(c, false) + flags.AddBackupIDFlag(c, false) + } + + return c +} + +// ------------------------------------------------------------------------------------------------ +// backup create +// ------------------------------------------------------------------------------------------------ + +// `corso backup create chats [...]` +func teamschatsCreateCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Aliases: []string{teamsServiceCommand}, + Short: "Backup M365 Chats data", + RunE: createTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teamschats backup. +func createTeamsChatsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := validateTeamsChatsBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil { + return err + } + + r, acct, err := utils.AccountConnectAndWriteRepoConfig( + ctx, + cmd, + path.TeamsChatsService) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + // TODO: log/print recoverable errors + errs := fault.New(false) + + svcCli, err := m365.NewM365Client(ctx, *acct) + if err != nil { + return Only(ctx, clues.Stack(err)) + } + + ins, err := svcCli.AC.Users().GetAllIDsAndNames(ctx, errs) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 teamschats")) + } + + sel := teamschatsBackupCreateSelectors(ctx, ins, flags.UserFV, flags.CategoryDataFV) + selectorSet := []selectors.Selector{} + + for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) { + selectorSet = append(selectorSet, discSel.Selector) + } + + return genericCreateCommand( + ctx, + r, + "Chats", + selectorSet, + ins) +} + +// ------------------------------------------------------------------------------------------------ +// backup list +// ------------------------------------------------------------------------------------------------ + +// `corso backup list teamschats [...]` +func teamschatsListCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "List the history of M365 Chats backups", + RunE: listTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// lists the history of backup operations +func listTeamsChatsCmd(cmd *cobra.Command, args []string) error { + return genericListCommand(cmd, flags.BackupIDFV, path.TeamsChatsService, args) +} + +// ------------------------------------------------------------------------------------------------ +// backup details +// ------------------------------------------------------------------------------------------------ + +// `corso backup details teamschats [...]` +func teamschatsDetailsCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "Shows the details of a M365 Chats backup", + RunE: detailsTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teamschats backup. +func detailsTeamsChatsCmd(cmd *cobra.Command, args []string) error { + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + return runDetailsTeamsChatsCmd(cmd) +} + +func runDetailsTeamsChatsCmd(cmd *cobra.Command) error { + ctx := cmd.Context() + opts := utils.MakeTeamsChatsOpts(cmd) + + sel := utils.IncludeTeamsChatsRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterTeamsChatsRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) + if err != nil { + return Only(ctx, err) + } + + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { + Info(ctx, selectors.ErrorNoMatchingItems) + } + + return nil +} + +// ------------------------------------------------------------------------------------------------ +// backup delete +// ------------------------------------------------------------------------------------------------ + +// `corso backup delete teamschats [...]` +func teamschatsDeleteCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "Delete backed-up M365 Chats data", + RunE: deleteTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// deletes an teamschats backup. +func deleteTeamsChatsCmd(cmd *cobra.Command, args []string) error { + backupIDValue := []string{} + + if len(flags.BackupIDsFV) > 0 { + backupIDValue = flags.BackupIDsFV + } else if len(flags.BackupIDFV) > 0 { + backupIDValue = append(backupIDValue, flags.BackupIDFV) + } else { + return clues.New("either --backup or --backups flag is required") + } + + return genericDeleteCommand(cmd, path.TeamsChatsService, "TeamsChats", backupIDValue, args) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func validateTeamsChatsBackupCreateFlags(teamschats, cats []string) error { + if len(teamschats) == 0 { + return clues.New( + "requires one or more --" + + flags.UserFN + " ids, or the wildcard --" + + flags.UserFN + " *") + } + + msg := fmt.Sprintf( + " is an unrecognized data type; only %s is supported", + flags.DataChats) + + allowedCats := utils.TeamsChatsAllowedCategories() + + for _, d := range cats { + if _, ok := allowedCats[d]; !ok { + return clues.New(d + msg) + } + } + + return nil +} + +func teamschatsBackupCreateSelectors( + ctx context.Context, + ins idname.Cacher, + users, cats []string, +) *selectors.TeamsChatsBackup { + if filters.PathContains(users).Compare(flags.Wildcard) { + return includeAllTeamsChatsWithCategories(ins, cats) + } + + sel := selectors.NewTeamsChatsBackup(slices.Clone(users)) + + return utils.AddTeamsChatsCategories(sel, cats) +} + +func includeAllTeamsChatsWithCategories(ins idname.Cacher, categories []string) *selectors.TeamsChatsBackup { + return utils.AddTeamsChatsCategories(selectors.NewTeamsChatsBackup(ins.IDs()), categories) +} diff --git a/src/cli/backup/teamschats_e2e_test.go b/src/cli/backup/teamschats_e2e_test.go new file mode 100644 index 000000000..2e829e628 --- /dev/null +++ b/src/cli/backup/teamschats_e2e_test.go @@ -0,0 +1,631 @@ +package backup_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli" + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/print" + cliTD "github.com/alcionai/corso/src/cli/testdata" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/config" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" +) + +// --------------------------------------------------------------------------- +// tests that require no existing backups +// --------------------------------------------------------------------------- + +type NoBackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + its intgTesterSetup +} + +func TestNoBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})}) +} + +func (suite *NoBackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) +} + +func (suite *NoBackupTeamsChatsE2ESuite) TestTeamsChatsBackupListCmd_noBackups() { + t := suite.T() + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + cmd.SetErr(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + result := suite.dpnd.recorder.String() + + // as an offhand check: the result should contain the m365 teamschat id + assert.True(t, strings.HasSuffix(result, "No backups available\n")) +} + +// --------------------------------------------------------------------------- +// tests with no prior backup +// --------------------------------------------------------------------------- + +type BackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + its intgTesterSetup +} + +func TestBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})}) +} + +func (suite *BackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) +} + +func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_chats() { + runTeamsChatsBackupCategoryTest(suite, flags.DataChats) +} + +func runTeamsChatsBackupCategoryTest(suite *BackupTeamsChatsE2ESuite, category string) { + recorder := strings.Builder{} + recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd, ctx := buildTeamsChatsBackupCmd( + ctx, + suite.dpnd.configFilePath, + suite.its.user.ID, + category, + &recorder) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + result := recorder.String() + t.Log("backup results", result) +} + +func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_teamschatNotFound_chats() { + runTeamsChatsBackupTeamsChatNotFoundTest(suite, flags.DataChats) +} + +func runTeamsChatsBackupTeamsChatNotFoundTest(suite *BackupTeamsChatsE2ESuite, category string) { + recorder := strings.Builder{} + recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd, ctx := buildTeamsChatsBackupCmd( + ctx, + suite.dpnd.configFilePath, + "foo@not-there.com", + category, + &recorder) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) + assert.Contains( + t, + err.Error(), + "not found", + "error missing user not found") + assert.NotContains(t, err.Error(), "runtime error", "panic happened") + + t.Logf("backup error message: %s", err.Error()) + + result := recorder.String() + t.Log("backup results", result) +} + +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAzureClientIDFlag() { + t := suite.T() + ctx, flush := tester.NewContext(t) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--azure-client-id", "invalid-value") + cli.BuildCommandTree(cmd) + + cmd.SetErr(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_fromConfigFile() { + t := suite.T() + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) +} + +// AWS flags +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAWSFlags() { + t := suite.T() + ctx, flush := tester.NewContext(t) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--aws-access-key", "invalid-value", + "--aws-secret-access-key", "some-invalid-value") + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + // since invalid aws creds are explicitly set, should see a failure + require.Error(t, err, clues.ToCore(err)) +} + +// --------------------------------------------------------------------------- +// tests prepared with a previous backup +// --------------------------------------------------------------------------- + +type PreparedBackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + backupOps map[path.CategoryType]string + its intgTesterSetup +} + +func TestPreparedBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &PreparedBackupTeamsChatsE2ESuite{ + Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) + suite.backupOps = make(map[path.CategoryType]string) + + var ( + teamschats = []string{suite.its.user.ID} + ins = idname.NewCache(map[string]string{suite.its.user.ID: suite.its.user.ID}) + cats = []path.CategoryType{ + path.ChatsCategory, + } + ) + + for _, set := range cats { + var ( + sel = selectors.NewTeamsChatsBackup(teamschats) + scopes []selectors.TeamsChatsScope + ) + + switch set { + case path.ChatsCategory: + scopes = selTD.TeamsChatsBackupChatScope(sel) + } + + sel.Include(scopes) + + bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins) + require.NoError(t, err, clues.ToCore(err)) + + err = bop.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + + bIDs := string(bop.Results.BackupID) + + // sanity check, ensure we can find the backup and its details immediately + b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID)) + require.NoError(t, err, "retrieving recent backup by ID") + require.Equal(t, bIDs, string(b.ID), "repo backup matches results id") + + _, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs) + require.NoError(t, errs.Failure(), "retrieving recent backup details by ID") + require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID") + require.Equal(t, bIDs, string(b.ID), "repo details matches results id") + + suite.backupOps[set] = string(b.ID) + } +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_chats() { + runTeamsChatsListCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsListCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + assert.Contains(t, result, suite.backupOps[category]) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_singleID_chats() { + runTeamsChatsListSingleCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsListSingleCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + bID := suite.backupOps[category] + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", string(bID)) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + assert.Contains(t, result, bID) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_badID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", "smarfs") + cli.BuildCommandTree(cmd) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsDetailsCmd_chats() { + runTeamsChatsDetailsCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsDetailsCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + bID := suite.backupOps[category] + + // fetch the details from the repo first + deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID)) + require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure())) + require.Empty(t, errs.Recovered()) + + cmd := cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupFN, string(bID)) + cli.BuildCommandTree(cmd) + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + + i := 0 + foundFolders := 0 + + for _, ent := range deets.Entries { + // Skip folders as they don't mean anything to the end teamschat. + if ent.Folder != nil { + foundFolders++ + continue + } + + suite.Run(fmt.Sprintf("detail %d", i), func() { + assert.Contains(suite.T(), result, ent.ShortRef) + }) + + i++ + } + + // We only backup the default folder for each category so there should be at + // least that folder (we don't make details entries for prefix folders). + assert.GreaterOrEqual(t, foundFolders, 1) +} + +// --------------------------------------------------------------------------- +// tests for deleting backups +// --------------------------------------------------------------------------- + +type BackupDeleteTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + backupOps [3]operations.BackupOperation +} + +func TestBackupDeleteTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupDeleteTeamsChatsE2ESuite{ + Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) + + m365TeamsChatID := tconfig.M365TeamID(t) + teamschats := []string{m365TeamsChatID} + + // some tests require an existing backup + sel := selectors.NewTeamsChatsBackup(teamschats) + sel.Include(selTD.TeamsChatsBackupChatScope(sel)) + + for i := 0; i < cap(suite.backupOps); i++ { + backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector) + require.NoError(t, err, clues.ToCore(err)) + + suite.backupOps[i] = backupOp + + err = suite.backupOps[i].Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + } +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupIDsFN, + fmt.Sprintf("%s,%s", + string(suite.backupOps[0].Results.BackupID), + string(suite.backupOps[1].Results.BackupID))) + cli.BuildCommandTree(cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // a follow-up details call should fail, due to the backup ID being deleted + cmd = cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backups", string(suite.backupOps[0].Results.BackupID)) + cli.BuildCommandTree(cmd) + + err = cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_SingleID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupFN, + string(suite.backupOps[2].Results.BackupID)) + cli.BuildCommandTree(cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // a follow-up details call should fail, due to the backup ID being deleted + cmd = cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", string(suite.backupOps[2].Results.BackupID)) + cli.BuildCommandTree(cmd) + + err = cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_UnknownID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupIDsFN, uuid.NewString()) + cli.BuildCommandTree(cmd) + + // unknown backupIDs should error since the modelStore can't find the backup + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_NoBackupID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + // empty backupIDs should error since no data provided + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func buildTeamsChatsBackupCmd( + ctx context.Context, + configFile, resource, category string, + recorder *strings.Builder, +) (*cobra.Command, context.Context) { + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--"+flags.ConfigFileFN, configFile, + "--"+flags.UserFN, resource, + "--"+flags.CategoryDataFN, category) + cli.BuildCommandTree(cmd) + cmd.SetOut(recorder) + + return cmd, print.SetRootCmd(ctx, cmd) +} diff --git a/src/cli/backup/teamschats_test.go b/src/cli/backup/teamschats_test.go new file mode 100644 index 000000000..177cc29e7 --- /dev/null +++ b/src/cli/backup/teamschats_test.go @@ -0,0 +1,248 @@ +package backup + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + flagsTD "github.com/alcionai/corso/src/cli/flags/testdata" + cliTD "github.com/alcionai/corso/src/cli/testdata" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" +) + +type TeamsChatsUnitSuite struct { + tester.Suite +} + +func TestTeamsChatsUnitSuite(t *testing.T) { + suite.Run(t, &TeamsChatsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *TeamsChatsUnitSuite) TestAddTeamsChatsCommands() { + expectUse := teamschatsServiceCommand + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + { + name: "create teamschats", + use: createCommand, + expectUse: expectUse + " " + teamschatsServiceCommandCreateUseSuffix, + expectShort: teamschatsCreateCmd().Short, + expectRunE: createTeamsChatsCmd, + }, + { + name: "list teamschats", + use: listCommand, + expectUse: expectUse, + expectShort: teamschatsListCmd().Short, + expectRunE: listTeamsChatsCmd, + }, + { + name: "details teamschats", + use: detailsCommand, + expectUse: expectUse + " " + teamschatsServiceCommandDetailsUseSuffix, + expectShort: teamschatsDetailsCmd().Short, + expectRunE: detailsTeamsChatsCmd, + }, + { + name: "delete teamschats", + use: deleteCommand, + expectUse: expectUse + " " + teamschatsServiceCommandDeleteUseSuffix, + expectShort: teamschatsDeleteCmd().Short, + expectRunE: deleteTeamsChatsCmd, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + c := addTeamsChatsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + }) + } +} + +func (suite *TeamsChatsUnitSuite) TestValidateTeamsChatsBackupCreateFlags() { + table := []struct { + name string + cats []string + expect assert.ErrorAssertionFunc + }{ + { + name: "none", + cats: []string{}, + expect: assert.NoError, + }, + { + name: "chats", + cats: []string{flags.DataChats}, + expect: assert.NoError, + }, + { + name: "all allowed", + cats: []string{ + flags.DataChats, + }, + expect: assert.NoError, + }, + { + name: "bad inputs", + cats: []string{"foo"}, + expect: assert.Error, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + err := validateTeamsChatsBackupCreateFlags([]string{"*"}, test.cats) + test.expect(suite.T(), err, clues.ToCore(err)) + }) + } +} + +func (suite *TeamsChatsUnitSuite) TestBackupCreateFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: createCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput), + "--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.TeamsChatsCategoryDataInput), + }, + flagsTD.PreparedGenericBackupFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + opts := utils.MakeTeamsChatsOpts(cmd) + co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) + + assert.Equal(t, control.FailFast, co.FailureHandling) + assert.True(t, co.ToggleFeatures.DisableIncrementals) + assert.True(t, co.ToggleFeatures.ForceItemDataDownload) + + assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users) + flagsTD.AssertGenericBackupFlags(t, cmd) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupListFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: listCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + }, + flagsTD.PreparedBackupListFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + flagsTD.AssertBackupListFlags(t, cmd) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupDetailsFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: detailsCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + "--" + flags.SkipReduceFN, + }, + flagsTD.PreparedTeamsChatsFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + co := utils.Control() + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + assert.True(t, co.SkipReduce) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) + flagsTD.AssertTeamsChatsFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupDeleteFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: deleteCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + }, + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} diff --git a/src/cli/flags/teamschats.go b/src/cli/flags/teamschats.go new file mode 100644 index 000000000..39fde2276 --- /dev/null +++ b/src/cli/flags/teamschats.go @@ -0,0 +1,13 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ( + DataChats = "chats" +) + +func AddTeamsChatsDetailsAndRestoreFlags(cmd *cobra.Command) { + // TODO: add details flags +} diff --git a/src/cli/flags/testdata/flags.go b/src/cli/flags/testdata/flags.go index 26ea8085b..1fe8a3c1d 100644 --- a/src/cli/flags/testdata/flags.go +++ b/src/cli/flags/testdata/flags.go @@ -21,6 +21,7 @@ var ( ExchangeCategoryDataInput = []string{"email", "events", "contacts"} SharepointCategoryDataInput = []string{"files", "lists", "pages"} GroupsCategoryDataInput = []string{"files", "lists", "pages", "messages"} + TeamsChatsCategoryDataInput = []string{"chats"} ChannelInput = []string{"channel1", "channel2"} MessageInput = []string{"message1", "message2"} diff --git a/src/cli/flags/testdata/teamschats.go b/src/cli/flags/testdata/teamschats.go new file mode 100644 index 000000000..9203ad586 --- /dev/null +++ b/src/cli/flags/testdata/teamschats.go @@ -0,0 +1,25 @@ +package testdata + +import ( + "testing" + + "github.com/spf13/cobra" +) + +func PreparedTeamsChatsFlags() []string { + return []string{ + // FIXME: populate when adding filters + // "--" + flags.ChatCreatedAfterFN, ChatCreatedAfterInput, + // "--" + flags.ChatCreatedBeforeFN, ChatCreatedBeforeInput, + // "--" + flags.ChatLastMessageAfterFN, ChatLastMessageAfterInput, + // "--" + flags.ChatLastMessageBeforeFN, ChatLastMessageBeforeInput, + } +} + +func AssertTeamsChatsFlags(t *testing.T, cmd *cobra.Command) { + // FIXME: populate when adding filters + // assert.Equal(t, ChatCreatedAfterInput, flags.ChatCreatedAfterFV) + // assert.Equal(t, ChatCreatedBeforeInput, flags.ChatCreatedBeforeFV) + // assert.Equal(t, ChatLastMessageAfterInput, flags.ChatLastMessageAfterFV) + // assert.Equal(t, ChatLastMessageBeforeInput, flags.ChatLastMessageBeforeFV) +} diff --git a/src/cli/utils/teamschats.go b/src/cli/utils/teamschats.go new file mode 100644 index 000000000..1467949a3 --- /dev/null +++ b/src/cli/utils/teamschats.go @@ -0,0 +1,101 @@ +package utils + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/selectors" +) + +type TeamsChatsOpts struct { + Users []string + + ExportCfg ExportCfgOpts + + Populated flags.PopulatedFlags +} + +func TeamsChatsAllowedCategories() map[string]struct{} { + return map[string]struct{}{ + flags.DataChats: {}, + } +} + +func AddTeamsChatsCategories(sel *selectors.TeamsChatsBackup, cats []string) *selectors.TeamsChatsBackup { + if len(cats) == 0 { + sel.Include(sel.AllData()) + } + + for _, d := range cats { + switch d { + case flags.DataChats: + sel.Include(sel.Chats(selectors.Any())) + } + } + + return sel +} + +func MakeTeamsChatsOpts(cmd *cobra.Command) TeamsChatsOpts { + return TeamsChatsOpts{ + Users: flags.UserFV, + + ExportCfg: makeExportCfgOpts(cmd), + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +// ValidateTeamsChatsRestoreFlags checks common flags for correctness and interdependencies +func ValidateTeamsChatsRestoreFlags(backupID string, opts TeamsChatsOpts, isRestore bool) error { + if len(backupID) == 0 { + return clues.New("a backup ID is required") + } + + // restore isn't currently supported + if isRestore { + return clues.New("restore not supported") + } + + return nil +} + +// AddTeamsChatsFilter adds the scope of the provided values to the selector's +// filter set +func AddTeamsChatsFilter( + sel *selectors.TeamsChatsRestore, + v string, + f func(string) []selectors.TeamsChatsScope, +) { + if len(v) == 0 { + return + } + + sel.Filter(f(v)) +} + +// IncludeTeamsChatsRestoreDataSelectors builds the common data-selector +// inclusions for teamschats commands. +func IncludeTeamsChatsRestoreDataSelectors(ctx context.Context, opts TeamsChatsOpts) *selectors.TeamsChatsRestore { + users := opts.Users + + if len(opts.Users) == 0 { + users = selectors.Any() + } + + return selectors.NewTeamsChatsRestore(users) +} + +// FilterTeamsChatsRestoreInfoSelectors builds the common info-selector filters. +func FilterTeamsChatsRestoreInfoSelectors( + sel *selectors.TeamsChatsRestore, + opts TeamsChatsOpts, +) { + // TODO: populate when adding filters +} diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index b6bab4120..5ea76dd4c 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -13,7 +13,10 @@ import ( "github.com/alcionai/corso/src/internal/m365/service/groups" "github.com/alcionai/corso/src/internal/m365/service/onedrive" "github.com/alcionai/corso/src/internal/m365/service/sharepoint" + "github.com/alcionai/corso/src/internal/m365/service/teamschats" + "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/account" bupMD "github.com/alcionai/corso/src/pkg/backup/metadata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -22,9 +25,33 @@ import ( "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +type backupHandler interface { + produceBackupCollectionser +} + +type produceBackupCollectionser interface { + ProduceBackupCollections( + ctx context.Context, + bpc inject.BackupProducerConfig, + ac api.Client, + creds account.M365Config, + su support.StatusUpdater, + counter *count.Bus, + errs *fault.Bus, + ) ( + collections []data.BackupCollection, + excludeItems *prefixmatcher.StringSetMatcher, + // canUsePreviousBacukp can be always returned true for impelementations + // that always return a tombstone collection when the metadata read fails + canUsePreviousBackup bool, + err error, + ) +} + // --------------------------------------------------------------------------- // Data Collections // --------------------------------------------------------------------------- @@ -63,67 +90,40 @@ func (ctrl *Controller) ProduceBackupCollections( canUsePreviousBackup bool ) + var handler backupHandler + switch service { case path.ExchangeService: - colls, excludeItems, canUsePreviousBackup, err = exchange.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = exchange.NewBackup() case path.OneDriveService: - colls, excludeItems, canUsePreviousBackup, err = onedrive.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = onedrive.NewBackup() case path.SharePointService: - colls, excludeItems, canUsePreviousBackup, err = sharepoint.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = sharepoint.NewBackup() case path.GroupsService: - colls, excludeItems, err = groups.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = groups.NewBackup() - // canUsePreviousBacukp can be always returned true for groups as we - // return a tombstone collection in case the metadata read fails - canUsePreviousBackup = true + case path.TeamsChatsService: + handler = teamschats.NewBackup() default: return nil, nil, false, clues.Wrap(clues.NewWC(ctx, service.String()), "service not supported") } + colls, excludeItems, canUsePreviousBackup, err = handler.ProduceBackupCollections( + ctx, + bpc, + ctrl.AC, + ctrl.credentials, + ctrl.UpdateStatus, + counter, + errs) + if err != nil { + return nil, nil, false, err + } + for _, c := range colls { // kopia doesn't stream Items() from deleted collections, // and so they never end up calling the UpdateStatus closer. @@ -153,25 +153,27 @@ func (ctrl *Controller) IsServiceEnabled( return sharepoint.IsServiceEnabled(ctx, ctrl.AC.Sites(), resourceOwner) case path.GroupsService: return groups.IsServiceEnabled(ctx, ctrl.AC.Groups(), resourceOwner) + case path.TeamsChatsService: + return teamschats.IsServiceEnabled(ctx, ctrl.AC.Users(), resourceOwner) } return false, clues.Wrap(clues.NewWC(ctx, service.String()), "service not supported") } -func verifyBackupInputs(sels selectors.Selector, cachedIDs []string) error { +func verifyBackupInputs(sel selectors.Selector, cachedIDs []string) error { var ids []string - switch sels.Service { + switch sel.Service { case selectors.ServiceExchange, selectors.ServiceOneDrive: // Exchange and OneDrive user existence now checked in checkServiceEnabled. return nil - case selectors.ServiceSharePoint, selectors.ServiceGroups: + case selectors.ServiceSharePoint, selectors.ServiceGroups, selectors.ServiceTeamsChats: ids = cachedIDs } - if !filters.Contains(ids).Compare(sels.ID()) { - return clues.Stack(core.ErrNotFound).With("selector_protected_resource", sels.DiscreteOwner) + if !filters.Contains(ids).Compare(sel.ID()) { + return clues.Stack(core.ErrNotFound).With("selector_protected_resource", sel.ID()) } return nil diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 5df7d5fff..2a95edda1 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -139,7 +139,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { Selector: sel, } - collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( + collections, excludes, canUsePreviousBackup, err := exchange.NewBackup().ProduceBackupCollections( ctx, bpc, suite.ac, @@ -309,7 +309,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { Selector: sel, } - collections, excludes, canUsePreviousBackup, err := sharepoint.ProduceBackupCollections( + collections, excludes, canUsePreviousBackup, err := sharepoint.NewBackup().ProduceBackupCollections( ctx, bpc, suite.ac, diff --git a/src/internal/m365/collection/teamschats/backup.go b/src/internal/m365/collection/teamschats/backup.go new file mode 100644 index 000000000..732d32286 --- /dev/null +++ b/src/internal/m365/collection/teamschats/backup.go @@ -0,0 +1,165 @@ +package teamschats + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/backup/metadata" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +func CreateCollections[I chatsItemer]( + ctx context.Context, + bpc inject.BackupProducerConfig, + bh backupHandler[I], + tenantID string, + scope selectors.TeamsChatsScope, + statusUpdater support.StatusUpdater, + useLazyReader bool, + counter *count.Bus, + errs *fault.Bus, +) ([]data.BackupCollection, bool, error) { + var ( + category = scope.Category().PathType() + qp = graph.QueryParams{ + Category: category, + ProtectedResource: bpc.ProtectedResource, + TenantID: tenantID, + } + ) + + cc := api.CallConfig{ + CanMakeDeltaQueries: false, + } + + container, err := bh.getContainer(ctx, cc) + if err != nil { + return nil, false, clues.Stack(err) + } + + counter.Add(count.Containers, 1) + + collection, err := populateCollection[I]( + ctx, + qp, + bh, + statusUpdater, + container, + scope, + useLazyReader, + bpc.Options, + counter, + errs) + if err != nil { + return nil, false, clues.Wrap(err, "filling collections") + } + + collections := []data.BackupCollection{collection} + + metadataPrefix, err := path.BuildMetadata( + qp.TenantID, + qp.ProtectedResource.ID(), + path.TeamsChatsService, + qp.Category, + false) + if err != nil { + return nil, false, clues.WrapWC(ctx, err, "making metadata path prefix"). + Label(count.BadPathPrefix) + } + + metadataCollection, err := graph.MakeMetadataCollection( + metadataPrefix, + // no deltas or previousPaths are used here; we store empty files instead + []graph.MetadataCollectionEntry{ + graph.NewMetadataEntry(metadata.PreviousPathFileName, map[string]string{}), + graph.NewMetadataEntry(metadata.DeltaURLsFileName, map[string]string{}), + }, + statusUpdater, + counter.Local()) + if err != nil { + return nil, false, clues.WrapWC(ctx, err, "making metadata collection") + } + + collections = append(collections, metadataCollection) + + // no deltas involved in this category, so canUsePrevBackups is always true. + return collections, true, nil +} + +func populateCollection[I chatsItemer]( + ctx context.Context, + qp graph.QueryParams, + bh backupHandler[I], + statusUpdater support.StatusUpdater, + container container[I], + scope selectors.TeamsChatsScope, + useLazyReader bool, + ctrlOpts control.Options, + counter *count.Bus, + errs *fault.Bus, +) (data.BackupCollection, error) { + var ( + cl = counter.Local() + collection data.BackupCollection + err error + ) + + ctx = clues.AddLabelCounter(ctx, cl.PlainAdder()) + cc := api.CallConfig{ + CanMakeDeltaQueries: false, + } + + items, err := bh.getItemIDs(ctx, cc) + if err != nil { + errs.AddRecoverable(ctx, clues.Stack(err)) + return collection, clues.Stack(errs.Failure()).OrNil() + } + + // Only create a collection if the path matches the scope. + includedItems := []I{} + + for _, item := range items { + if !bh.includeItem(item, scope) { + cl.Inc(count.SkippedItems) + continue + } + + includedItems = append(includedItems, item) + } + + cl.Add(count.ItemsAdded, int64(len(includedItems))) + + p, err := bh.CanonicalPath() + if err != nil { + err = clues.StackWC(ctx, err).Label(count.BadCollPath) + errs.AddRecoverable(ctx, err) + + return collection, clues.Stack(errs.Failure()).OrNil() + } + + collection = NewCollection( + data.NewBaseCollection( + p, + p, + container.humanLocation.Builder(), + ctrlOpts, + false, + cl), + bh, + qp.ProtectedResource.ID(), + includedItems, + container, + statusUpdater) + + return collection, clues.Stack(errs.Failure()).OrNil() +} diff --git a/src/internal/m365/collection/teamschats/backup_test.go b/src/internal/m365/collection/teamschats/backup_test.go new file mode 100644 index 000000000..bbb8ee0d5 --- /dev/null +++ b/src/internal/m365/collection/teamschats/backup_test.go @@ -0,0 +1,366 @@ +package teamschats + +import ( + "context" + "testing" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + inMock "github.com/alcionai/corso/src/internal/common/idname/mock" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/teamschats/testdata" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/errs/core" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +// --------------------------------------------------------------------------- +// mocks +// --------------------------------------------------------------------------- + +var _ backupHandler[models.Chatable] = &mockBackupHandler{} + +//lint:ignore U1000 false linter issue due to generics +type mockBackupHandler struct { + chatsErr error + chats []models.Chatable + chatMessagesErr error + chatMessages map[string][]models.ChatMessageable + info map[string]*details.TeamsChatsInfo + getMessageErr map[string]error + doNotInclude bool +} + +//lint:ignore U1000 false linter issue due to generics +func (bh mockBackupHandler) augmentItemInfo( + *details.TeamsChatsInfo, + models.Chatable, +) { + // no-op +} + +func (bh mockBackupHandler) container() container[models.Chatable] { + return chatContainer() +} + +//lint:ignore U1000 required for interface compliance +func (bh mockBackupHandler) getContainer( + context.Context, + api.CallConfig, +) (container[models.Chatable], error) { + return chatContainer(), nil +} + +func (bh mockBackupHandler) getItemIDs( + _ context.Context, + _ api.CallConfig, +) ([]models.Chatable, error) { + return bh.chats, bh.chatsErr +} + +//lint:ignore U1000 required for interface compliance +func (bh mockBackupHandler) includeItem( + models.Chatable, + selectors.TeamsChatsScope, +) bool { + return !bh.doNotInclude +} + +func (bh mockBackupHandler) CanonicalPath() (path.Path, error) { + return path.BuildPrefix( + "tenant", + "protectedResource", + path.TeamsChatsService, + path.ChatsCategory) +} + +//lint:ignore U1000 false linter issue due to generics +func (bh mockBackupHandler) getItem( + _ context.Context, + _ string, + itemID string, +) (models.Chatable, *details.TeamsChatsInfo, error) { + chat := models.NewChat() + + chat.SetId(ptr.To(itemID)) + chat.SetTopic(ptr.To(itemID)) + chat.SetMessages(bh.chatMessages[itemID]) + + return chat, bh.info[itemID], bh.getMessageErr[itemID] +} + +// --------------------------------------------------------------------------- +// Unit Suite +// --------------------------------------------------------------------------- + +type BackupUnitSuite struct { + tester.Suite + creds account.M365Config +} + +func TestServiceIteratorsUnitSuite(t *testing.T) { + suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *BackupUnitSuite) SetupSuite() { + a := tconfig.NewFakeM365Account(suite.T()) + m365, err := a.M365Config() + require.NoError(suite.T(), err, clues.ToCore(err)) + suite.creds = m365 +} + +func (suite *BackupUnitSuite) TestPopulateCollections() { + var ( + qp = graph.QueryParams{ + Category: path.ChatsCategory, // doesn't matter which one we use. + ProtectedResource: inMock.NewProvider("user_id", "user_name"), + TenantID: suite.creds.AzureTenantID, + } + statusUpdater = func(*support.ControllerOperationStatus) {} + ) + + table := []struct { + name string + mock mockBackupHandler + expectErr require.ErrorAssertionFunc + expectColl require.ValueAssertionFunc + }{ + { + name: "happy path, one chat", + mock: mockBackupHandler{ + chats: testdata.StubChats("one"), + chatMessages: map[string][]models.ChatMessageable{ + "one": testdata.StubChatMessages("msg-one"), + }, + }, + expectErr: require.NoError, + expectColl: require.NotNil, + }, + { + name: "happy path, many chats", + mock: mockBackupHandler{ + chats: testdata.StubChats("one", "two"), + chatMessages: map[string][]models.ChatMessageable{ + "one": testdata.StubChatMessages("msg-one"), + "two": testdata.StubChatMessages("msg-two"), + }, + }, + expectErr: require.NoError, + expectColl: require.NotNil, + }, + { + name: "no chats pass scope", + mock: mockBackupHandler{ + chats: testdata.StubChats("one"), + doNotInclude: true, + }, + expectErr: require.NoError, + expectColl: require.NotNil, + }, + { + name: "no chats", + mock: mockBackupHandler{}, + expectErr: require.NoError, + expectColl: require.NotNil, + }, + { + name: "no chat messages", + mock: mockBackupHandler{ + chats: testdata.StubChats("one"), + }, + expectErr: require.NoError, + expectColl: require.NotNil, + }, + { + name: "err: deleted in flight", + mock: mockBackupHandler{ + chats: testdata.StubChats("one"), + chatsErr: core.ErrNotFound, + }, + expectErr: require.Error, + expectColl: require.Nil, + }, + { + name: "err enumerating chats", + mock: mockBackupHandler{ + chats: testdata.StubChats("one"), + chatsErr: assert.AnError, + }, + expectErr: require.Error, + expectColl: require.Nil, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ctrlOpts := control.Options{FailureHandling: control.FailFast} + + result, err := populateCollection( + ctx, + qp, + test.mock, + statusUpdater, + test.mock.container(), + selectors.NewTeamsChatsBackup(nil).Chats(selectors.Any())[0], + false, + ctrlOpts, + count.New(), + fault.New(true)) + test.expectErr(t, err, clues.ToCore(err)) + test.expectColl(t, result) + + if err != nil || result == nil { + return + } + + // collection assertions + + assert.NotEqual( + t, + result.FullPath().Service(), + path.TeamsChatsMetadataService, + "should not contain metadata collections") + assert.NotEqual(t, result.State(), data.DeletedState, "no tombstones should be produced") + assert.Equal(t, result.State(), data.NotMovedState) + assert.False(t, result.DoNotMergeItems(), "doNotMergeItems should always be false") + }) + } +} + +// --------------------------------------------------------------------------- +// Integration tests +// --------------------------------------------------------------------------- + +type BackupIntgSuite struct { + tester.Suite + resource string + tenantID string + ac api.Client +} + +func TestBackupIntgSuite(t *testing.T) { + suite.Run(t, &BackupIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *BackupIntgSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, true, 4) + + suite.resource = tconfig.M365TeamID(t) + + acct := tconfig.NewM365Account(t) + creds, err := acct.M365Config() + require.NoError(t, err, clues.ToCore(err)) + + suite.ac, err = api.NewClient( + creds, + control.DefaultOptions(), + count.New()) + require.NoError(t, err, clues.ToCore(err)) + + suite.tenantID = creds.AzureTenantID +} + +func (suite *BackupIntgSuite) TestCreateCollections() { + var ( + tenant = tconfig.M365TenantID(suite.T()) + protectedResource = tconfig.M365TeamID(suite.T()) + resources = []string{protectedResource} + handler = NewUsersChatsBackupHandler(tenant, protectedResource, suite.ac.Chats()) + ) + + tests := []struct { + name string + scope selectors.TeamsChatsScope + chatNames map[string]struct{} + }{ + { + name: "chat messages", + scope: selTD.TeamsChatsBackupChatScope(selectors.NewTeamsChatsBackup(resources))[0], + chatNames: map[string]struct{}{ + selTD.TestChatTopic: {}, + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ctrlOpts := control.DefaultOptions() + + sel := selectors.NewTeamsChatsBackup([]string{protectedResource}) + sel.Include(selTD.TeamsChatsBackupChatScope(sel)) + + bpc := inject.BackupProducerConfig{ + LastBackupVersion: version.NoBackup, + Options: ctrlOpts, + ProtectedResource: inMock.NewProvider(protectedResource, protectedResource), + Selector: sel.Selector, + } + + collections, _, err := CreateCollections( + ctx, + bpc, + handler, + suite.tenantID, + test.scope, + func(status *support.ControllerOperationStatus) {}, + false, + count.New(), + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, collections, "must have at least one collection") + + for _, c := range collections { + if c.FullPath().Service() == path.TeamsChatsMetadataService { + continue + } + + require.Empty(t, c.FullPath().Folder(false), "all items should be stored at the root") + + locp, ok := c.(data.LocationPather) + + if ok { + loc := locp.LocationPath().String() + require.Empty(t, loc, "no items should have locations") + } + } + + assert.Len(t, collections, 2, "should have the root folder collection and metadata collection") + }) + } +} diff --git a/src/internal/m365/collection/teamschats/chat_handler.go b/src/internal/m365/collection/teamschats/chat_handler.go new file mode 100644 index 000000000..93ef8316a --- /dev/null +++ b/src/internal/m365/collection/teamschats/chat_handler.go @@ -0,0 +1,102 @@ +package teamschats + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +var _ backupHandler[models.Chatable] = &usersChatsBackupHandler{} + +type usersChatsBackupHandler struct { + ac api.Chats + protectedResourceID string + tenantID string +} + +func NewUsersChatsBackupHandler( + tenantID, protectedResourceID string, + ac api.Chats, +) usersChatsBackupHandler { + return usersChatsBackupHandler{ + ac: ac, + protectedResourceID: protectedResourceID, + tenantID: tenantID, + } +} + +// chats have no containers. Everything is stored at the root. +// +//lint:ignore U1000 required for interface compliance +func (bh usersChatsBackupHandler) getContainer( + ctx context.Context, + _ api.CallConfig, +) (container[models.Chatable], error) { + return chatContainer(), nil +} + +//lint:ignore U1000 required for interface compliance +func (bh usersChatsBackupHandler) getItemIDs( + ctx context.Context, + cc api.CallConfig, +) ([]models.Chatable, error) { + return bh.ac.GetChats( + ctx, + bh.protectedResourceID, + cc) +} + +//lint:ignore U1000 required for interface compliance +func (bh usersChatsBackupHandler) includeItem( + ch models.Chatable, + scope selectors.TeamsChatsScope, +) bool { + // corner case: many Topics are empty, and empty inputs are automatically + // set to non-matching in the selectors code. This allows us to include + // everything without needing to check the topic value in that case. + if scope.IsAny(selectors.TeamsChatsChat) { + return true + } + + return scope.Matches(selectors.TeamsChatsChat, ptr.Val(ch.GetTopic())) +} + +func (bh usersChatsBackupHandler) CanonicalPath() (path.Path, error) { + return path.BuildPrefix( + bh.tenantID, + bh.protectedResourceID, + path.TeamsChatsService, + path.ChatsCategory) +} + +//lint:ignore U1000 false linter issue due to generics +func (bh usersChatsBackupHandler) getItem( + ctx context.Context, + userID string, + chatID string, +) (models.Chatable, *details.TeamsChatsInfo, error) { + // FIXME: should retrieve and populate all messages in the chat. + return nil, nil, clues.New("not implemented") +} + +//lint:ignore U1000 false linter issue due to generics +func (bh usersChatsBackupHandler) augmentItemInfo( + dgi *details.TeamsChatsInfo, + c models.Chatable, +) { + // no-op +} + +func chatContainer() container[models.Chatable] { + return container[models.Chatable]{ + storageDirFolders: path.Elements{}, + humanLocation: path.Elements{}, + } +} diff --git a/src/internal/m365/collection/teamschats/collection.go b/src/internal/m365/collection/teamschats/collection.go new file mode 100644 index 000000000..a7f25c601 --- /dev/null +++ b/src/internal/m365/collection/teamschats/collection.go @@ -0,0 +1,261 @@ +package teamschats + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/alcionai/clues" + kjson "github.com/microsoft/kiota-serialization-json-go" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/errs/core" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +var _ data.BackupCollection = &lazyFetchCollection[chatsItemer]{} + +const ( + collectionChannelBufferSize = 1000 + numberOfRetries = 4 +) + +// updateStatus is a utility function used to send the status update through +// the channel. +func updateStatus( + ctx context.Context, + statusUpdater support.StatusUpdater, + attempted int, + streamedItems int64, + totalBytes int64, + folderPath string, + err error, +) { + status := support.CreateStatus( + ctx, + support.Backup, + 1, + support.CollectionMetrics{ + Objects: attempted, + Successes: int(streamedItems), + Bytes: totalBytes, + }, + folderPath) + + logger.Ctx(ctx).Debugw("done streaming items", "status", status.String()) + + statusUpdater(status) +} + +// State of the collection is set as an observation of the current +// and previous paths. If the curr path is nil, the state is assumed +// to be deleted. If the prev path is nil, it is assumed newly created. +// If both are populated, then state is either moved (if they differ), +// or notMoved (if they match). +func NewCollection[I chatsItemer]( + baseCol data.BaseCollection, + getAndAugment getItemAndAugmentInfoer[I], + protectedResource string, + items []I, + contains container[I], + statusUpdater support.StatusUpdater, +) data.BackupCollection { + return &lazyFetchCollection[I]{ + BaseCollection: baseCol, + items: items, + contains: contains, + getAndAugment: getAndAugment, + statusUpdater: statusUpdater, + stream: make(chan data.Item, collectionChannelBufferSize), + protectedResource: protectedResource, + } +} + +// ----------------------------------------------------------------------------- +// lazyFetchCollection +// ----------------------------------------------------------------------------- + +type lazyFetchCollection[I chatsItemer] struct { + data.BaseCollection + protectedResource string + stream chan data.Item + + contains container[I] + + items []I + + getAndAugment getItemAndAugmentInfoer[I] + + statusUpdater support.StatusUpdater +} + +func (col *lazyFetchCollection[I]) Items( + ctx context.Context, + errs *fault.Bus, +) <-chan data.Item { + go col.streamItems(ctx, errs) + return col.stream +} + +func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.Bus) { + var ( + streamedItems int64 + wg sync.WaitGroup + progressMessage chan<- struct{} + el = errs.Local() + ) + + ctx = clues.Add(ctx, "category", col.Category().String()) + + defer func() { + close(col.stream) + logger.Ctx(ctx).Infow( + "finished stream backup collection items", + "stats", col.Counter.Values()) + + updateStatus( + ctx, + col.statusUpdater, + len(col.items), + streamedItems, + 0, + col.FullPath().Folder(false), + errs.Failure()) + }() + + if len(col.items) > 0 { + progressMessage = observe.CollectionProgress( + ctx, + col.Category().HumanString(), + col.LocationPath().Elements()) + defer close(progressMessage) + } + + semaphoreCh := make(chan struct{}, col.Opts().Parallelism.ItemFetch) + defer close(semaphoreCh) + + // add any new items + for _, item := range col.items { + if el.Failure() != nil { + break + } + + itemID := ptr.Val(item.GetId()) + modTime := ptr.Val(item.GetLastUpdatedDateTime()) + + wg.Add(1) + semaphoreCh <- struct{}{} + + go func(id string, modTime time.Time) { + defer wg.Done() + defer func() { <-semaphoreCh }() + + ictx := clues.Add( + ctx, + "item_id", id, + "parent_path", path.LoggableDir(col.LocationPath().String())) + + col.stream <- data.NewLazyItemWithInfo( + ictx, + &lazyItemGetter[I]{ + modTime: modTime, + getAndAugment: col.getAndAugment, + resourceID: col.protectedResource, + itemID: id, + containerIDs: col.FullPath().Folders(), + contains: col.contains, + parentPath: col.LocationPath().String(), + }, + id, + modTime, + col.Counter, + el) + + atomic.AddInt64(&streamedItems, 1) + + if progressMessage != nil { + progressMessage <- struct{}{} + } + }(itemID, modTime) + } + + wg.Wait() +} + +type lazyItemGetter[I chatsItemer] struct { + getAndAugment getItemAndAugmentInfoer[I] + resourceID string + itemID string + parentPath string + containerIDs path.Elements + modTime time.Time + contains container[I] +} + +func (lig *lazyItemGetter[I]) GetData( + ctx context.Context, + errs *fault.Bus, +) (io.ReadCloser, *details.ItemInfo, bool, error) { + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + item, info, err := lig.getAndAugment.getItem( + ctx, + lig.resourceID, + lig.itemID) + if err != nil { + // For items that were deleted in flight, add the skip label so that + // they don't lead to recoverable failures during backup. + if clues.HasLabel(err, graph.LabelStatus(http.StatusNotFound)) || errors.Is(err, core.ErrNotFound) { + logger.CtxErr(ctx, err).Info("item deleted in flight. skipping") + + // Returning delInFlight as true here for correctness, although the caller is going + // to ignore it since we are returning an error. + return nil, nil, true, clues.Wrap(err, "deleted item").Label(graph.LabelsSkippable) + } + + err = clues.WrapWC(ctx, err, "getting item data").Label(fault.LabelForceNoBackupCreation) + errs.AddRecoverable(ctx, err) + + return nil, nil, false, err + } + + lig.getAndAugment.augmentItemInfo(info, lig.contains.container) + + if err := writer.WriteObjectValue("", item); err != nil { + err = clues.WrapWC(ctx, err, "writing item to serializer").Label(fault.LabelForceNoBackupCreation) + errs.AddRecoverable(ctx, err) + + return nil, nil, false, err + } + + itemData, err := writer.GetSerializedContent() + if err != nil { + err = clues.WrapWC(ctx, err, "serializing item").Label(fault.LabelForceNoBackupCreation) + errs.AddRecoverable(ctx, err) + + return nil, nil, false, err + } + + info.ParentPath = lig.parentPath + // Update the mod time to what we already told kopia about. This is required + // for proper details merging. + info.Modified = lig.modTime + + return io.NopCloser(bytes.NewReader(itemData)), + &details.ItemInfo{TeamsChats: info}, + false, + nil +} diff --git a/src/internal/m365/collection/teamschats/collection_test.go b/src/internal/m365/collection/teamschats/collection_test.go new file mode 100644 index 000000000..15b0dac77 --- /dev/null +++ b/src/internal/m365/collection/teamschats/collection_test.go @@ -0,0 +1,401 @@ +package teamschats + +import ( + "bytes" + "context" + "io" + "slices" + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/readers" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/teamschats/testdata" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/errs/core" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type CollectionUnitSuite struct { + tester.Suite +} + +func TestCollectionUnitSuite(t *testing.T) { + suite.Run(t, &CollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *CollectionUnitSuite) TestPrefetchedItem_Reader() { + table := []struct { + name string + readData []byte + }{ + { + name: "HasData", + readData: []byte("test message"), + }, + { + name: "Empty", + readData: []byte{}, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ed, err := data.NewPrefetchedItemWithInfo( + io.NopCloser(bytes.NewReader(test.readData)), + "itemID", + details.ItemInfo{}) + require.NoError(t, err, clues.ToCore(err)) + + r, err := readers.NewVersionedRestoreReader(ed.ToReader()) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, readers.DefaultSerializationVersion, r.Format().Version) + assert.False(t, r.Format().DelInFlight) + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(r) + assert.NoError(t, err, "reading data: %v", clues.ToCore(err)) + assert.Equal(t, test.readData, buf.Bytes(), "read data") + assert.Equal(t, "itemID", ed.ID(), "item ID") + }) + } +} + +func (suite *CollectionUnitSuite) TestNewCollection_state() { + fooP, err := path.Build("t", "u", path.TeamsChatsService, path.ChatsCategory, false, "foo") + require.NoError(suite.T(), err, clues.ToCore(err)) + barP, err := path.Build("t", "u", path.TeamsChatsService, path.ChatsCategory, false, "bar") + require.NoError(suite.T(), err, clues.ToCore(err)) + + locPB := path.Builder{}.Append("human-readable") + + table := []struct { + name string + prev path.Path + curr path.Path + loc *path.Builder + expect data.CollectionState + }{ + { + name: "new", + curr: fooP, + loc: locPB, + expect: data.NewState, + }, + { + name: "not moved", + prev: fooP, + curr: fooP, + loc: locPB, + expect: data.NotMovedState, + }, + { + name: "moved", + prev: fooP, + curr: barP, + loc: locPB, + expect: data.MovedState, + }, + { + name: "deleted", + prev: fooP, + expect: data.DeletedState, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + c := NewCollection[models.Chatable]( + data.NewBaseCollection( + test.curr, + test.prev, + test.loc, + control.DefaultOptions(), + false, + count.New()), + nil, + "g", + nil, + container[models.Chatable]{}, + nil) + + assert.Equal(t, test.expect, c.State(), "collection state") + assert.Equal(t, test.curr, c.FullPath(), "full path") + assert.Equal(t, test.prev, c.PreviousPath(), "prev path") + + prefetch, ok := c.(*lazyFetchCollection[models.Chatable]) + require.True(t, ok, "collection type") + + assert.Equal(t, test.loc, prefetch.LocationPath(), "location path") + }) + } +} + +type getAndAugmentChat struct { + err error +} + +//lint:ignore U1000 false linter issue due to generics +func (m getAndAugmentChat) getItem( + _ context.Context, + _ string, + itemID string, +) (models.Chatable, *details.TeamsChatsInfo, error) { + chat := models.NewChat() + chat.SetId(ptr.To(itemID)) + chat.SetTopic(ptr.To(itemID)) + + return chat, &details.TeamsChatsInfo{}, m.err +} + +//lint:ignore U1000 false linter issue due to generics +func (getAndAugmentChat) augmentItemInfo(*details.TeamsChatsInfo, models.Chatable) { + // no-op +} + +func (suite *CollectionUnitSuite) TestLazyFetchCollection_Items_LazyFetch() { + var ( + t = suite.T() + statusUpdater = func(*support.ControllerOperationStatus) {} + ) + + fullPath, err := path.BuildPrefix("t", "pr", path.TeamsChatsService, path.ChatsCategory) + require.NoError(t, err, clues.ToCore(err)) + + locPath, err := path.BuildPrefix("t", "pr", path.TeamsChatsService, path.ChatsCategory) + require.NoError(t, err, clues.ToCore(err)) + + table := []struct { + name string + items []models.Chatable + expectItemCount int + // Items we want to trigger lazy reader on. + expectReads []string + }{ + { + name: "no items", + }, + { + name: "items", + items: testdata.StubChats("fisher", "flannigan", "fitzbog"), + expectItemCount: 3, + expectReads: []string{ + "fisher", + "flannigan", + "fitzbog", + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + var ( + t = suite.T() + errs = fault.New(true) + itemCount int + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + getterAugmenter := &getAndAugmentChat{} + + col := &lazyFetchCollection[models.Chatable]{ + BaseCollection: data.NewBaseCollection( + fullPath, + nil, + locPath.ToBuilder(), + control.DefaultOptions(), + false, + count.New()), + items: test.items, + contains: container[models.Chatable]{}, + getAndAugment: getterAugmenter, + stream: make(chan data.Item), + statusUpdater: statusUpdater, + } + + for item := range col.Items(ctx, errs) { + itemCount++ + + ok := slices.ContainsFunc(test.items, func(mc models.Chatable) bool { + return ptr.Val(mc.GetId()) == item.ID() + }) + + require.True(t, ok, "item must be either added or removed: %q", item.ID()) + assert.False(t, item.Deleted(), "additions should not be marked as deleted") + } + + assert.NoError(t, errs.Failure()) + assert.Equal( + t, + test.expectItemCount, + itemCount, + "should see all expected items") + }) + } +} + +func (suite *CollectionUnitSuite) TestLazyItem_GetDataErrors() { + var ( + parentPath = "" + now = time.Now() + ) + + table := []struct { + name string + getErr error + expectReadErrType error + }{ + { + name: "ReturnsErrorOnGenericGetError", + getErr: assert.AnError, + expectReadErrType: assert.AnError, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + m := getAndAugmentChat{ + err: test.getErr, + } + + li := data.NewLazyItemWithInfo( + ctx, + &lazyItemGetter[models.Chatable]{ + resourceID: "resourceID", + itemID: "itemID", + getAndAugment: &m, + modTime: now, + parentPath: parentPath, + }, + "itemID", + now, + count.New(), + fault.New(true)) + + assert.False(t, li.Deleted(), "item shouldn't be marked deleted") + assert.Equal(t, now, li.ModTime(), "item mod time") + + _, err := readers.NewVersionedRestoreReader(li.ToReader()) + assert.ErrorIs(t, err, test.expectReadErrType) + + // Should get some form of error when trying to get info. + _, err = li.Info() + assert.Error(t, err, "Info()") + }) + } +} + +func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlight() { + var ( + t = suite.T() + + parentPath = "" + now = time.Now() + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + m := getAndAugmentChat{ + err: core.ErrNotFound, + } + + li := data.NewLazyItemWithInfo( + ctx, + &lazyItemGetter[models.Chatable]{ + resourceID: "resourceID", + itemID: "itemID", + getAndAugment: &m, + modTime: now, + parentPath: parentPath, + }, + "itemID", + now, + count.New(), + fault.New(true)) + + assert.False(t, li.Deleted(), "item shouldn't be marked deleted") + assert.Equal( + t, + now, + li.ModTime(), + "item mod time") + + _, err := readers.NewVersionedRestoreReader(li.ToReader()) + assert.ErrorIs(t, err, core.ErrNotFound, "item should be marked deleted in flight") +} + +func (suite *CollectionUnitSuite) TestLazyItem() { + var ( + t = suite.T() + + parentPath = "" + now = time.Now() + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + m := getAndAugmentChat{} + + li := data.NewLazyItemWithInfo( + ctx, + &lazyItemGetter[models.Chatable]{ + resourceID: "resourceID", + itemID: "itemID", + getAndAugment: &m, + modTime: now, + parentPath: parentPath, + }, + "itemID", + now, + count.New(), + fault.New(true)) + + assert.False(t, li.Deleted(), "item shouldn't be marked deleted") + assert.Equal( + t, + now, + li.ModTime(), + "item mod time") + + r, err := readers.NewVersionedRestoreReader(li.ToReader()) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, readers.DefaultSerializationVersion, r.Format().Version) + assert.False(t, r.Format().DelInFlight) + + readData, err := io.ReadAll(r) + assert.NoError(t, err, "reading item data: %v", clues.ToCore(err)) + + assert.NotEmpty(t, readData, "read item data") + + info, err := li.Info() + assert.NoError(t, err, "getting item info: %v", clues.ToCore(err)) + + assert.Empty(t, parentPath) + assert.Equal(t, now, info.Modified()) +} diff --git a/src/internal/m365/collection/teamschats/debug.go b/src/internal/m365/collection/teamschats/debug.go new file mode 100644 index 000000000..9c870a10b --- /dev/null +++ b/src/internal/m365/collection/teamschats/debug.go @@ -0,0 +1,17 @@ +package teamschats + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/store" +) + +func DeserializeMetadataFiles( + ctx context.Context, + colls []data.RestoreCollection, +) ([]store.MetadataFile, error) { + return nil, clues.New("no metadata stored for this service/category") +} diff --git a/src/internal/m365/collection/teamschats/handlers.go b/src/internal/m365/collection/teamschats/handlers.go new file mode 100644 index 000000000..f1db7d32b --- /dev/null +++ b/src/internal/m365/collection/teamschats/handlers.go @@ -0,0 +1,93 @@ +package teamschats + +import ( + "context" + + "github.com/microsoft/kiota-abstractions-go/serialization" + + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +// itemer standardizes common behavior that can be expected from all +// items within a chats collection backup. +type chatsItemer interface { + serialization.Parsable + graph.GetIDer + graph.GetLastUpdatedDateTimer +} + +type backupHandler[I chatsItemer] interface { + getContainerer[I] + getItemAndAugmentInfoer[I] + getItemer[I] + getItemIDser[I] + includeItemer[I] + canonicalPather +} + +// gets the container for the resource +// within this handler set, only one container (the root) +// is expected +type getContainerer[I chatsItemer] interface { + getContainer( + ctx context.Context, + cc api.CallConfig, + ) (container[I], error) +} + +type getItemAndAugmentInfoer[I chatsItemer] interface { + getItemer[I] + augmentItemInfoer[I] +} + +type augmentItemInfoer[I chatsItemer] interface { + // augmentItemInfo completes the teamChatsInfo population with any data + // owned by the container and not accessible to the item. + augmentItemInfo(*details.TeamsChatsInfo, I) +} + +// gets all item IDs in the container +type getItemIDser[I chatsItemer] interface { + getItemIDs( + ctx context.Context, + cc api.CallConfig, + ) ([]I, error) +} + +type getItemer[I chatsItemer] interface { + getItem( + ctx context.Context, + protectedResource string, + itemID string, + ) (I, *details.TeamsChatsInfo, error) +} + +// includeItemer evaluates whether the item is included +// in the provided scope. +type includeItemer[I chatsItemer] interface { + includeItem( + i I, + scope selectors.TeamsChatsScope, + ) bool +} + +// canonicalPath constructs the service and category specific path for +// the given builder. The tenantID and protectedResourceID are assumed +// to be stored in the handler already. +type canonicalPather interface { + CanonicalPath() (path.Path, error) +} + +// --------------------------------------------------------------------------- +// Container management +// --------------------------------------------------------------------------- + +type container[I chatsItemer] struct { + storageDirFolders path.Elements + humanLocation path.Elements + container I +} diff --git a/src/internal/m365/collection/teamschats/testdata/chats.go b/src/internal/m365/collection/teamschats/testdata/chats.go new file mode 100644 index 000000000..9d2800910 --- /dev/null +++ b/src/internal/m365/collection/teamschats/testdata/chats.go @@ -0,0 +1,40 @@ +package testdata + +import ( + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func StubChats(ids ...string) []models.Chatable { + sl := make([]models.Chatable, 0, len(ids)) + + for _, id := range ids { + ch := models.NewChat() + ch.SetTopic(ptr.To(id)) + ch.SetId(ptr.To(id)) + + sl = append(sl, ch) + } + + return sl +} + +func StubChatMessages(ids ...string) []models.ChatMessageable { + sl := make([]models.ChatMessageable, 0, len(ids)) + + for _, id := range ids { + cm := models.NewChatMessage() + cm.SetId(ptr.To(uuid.NewString())) + + body := models.NewItemBody() + body.SetContent(ptr.To(id)) + + cm.SetBody(body) + + sl = append(sl, cm) + } + + return sl +} diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 4057b2424..536f24f78 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -112,7 +112,7 @@ func (ctrl *Controller) setResourceHandler( var rh *resourceGetter switch serviceInOperation { - case path.ExchangeService, path.OneDriveService: + case path.ExchangeService, path.OneDriveService, path.TeamsChatsService: rh = &resourceGetter{ enum: resource.Users, getter: ctrl.AC.Users(), diff --git a/src/internal/m365/debug.go b/src/internal/m365/debug.go index 038706e79..c90b39d01 100644 --- a/src/internal/m365/debug.go +++ b/src/internal/m365/debug.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/collection/exchange" "github.com/alcionai/corso/src/internal/m365/collection/groups" + "github.com/alcionai/corso/src/internal/m365/collection/teamschats" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/store" @@ -34,6 +35,8 @@ func (ctrl *Controller) DeserializeMetadataFiles( return drive.DeserializeMetadataFiles(ctx, colls, count.New()) case path.GroupsService, path.GroupsMetadataService: return groups.DeserializeMetadataFiles(ctx, colls) + case path.TeamsChatsService, path.TeamsChatsMetadataService: + return teamschats.DeserializeMetadataFiles(ctx, colls) default: return nil, clues.NewWC(ctx, "unrecognized service").With("service", service) } diff --git a/src/internal/m365/service/exchange/backup.go b/src/internal/m365/service/exchange/backup.go index 879160375..999b2441d 100644 --- a/src/internal/m365/service/exchange/backup.go +++ b/src/internal/m365/service/exchange/backup.go @@ -19,9 +19,17 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +type exchangeBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *exchangeBackup { + return &exchangeBackup{} +} + // ProduceBackupCollections returns a DataCollection which the caller can // use to read mailbox data out for the specified user -func ProduceBackupCollections( +func (exchangeBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/groups/backup.go b/src/internal/m365/service/groups/backup.go index 07b1e1ada..9f5ac8e85 100644 --- a/src/internal/m365/service/groups/backup.go +++ b/src/internal/m365/service/groups/backup.go @@ -34,7 +34,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type groupsBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *groupsBackup { + return &groupsBackup{} +} + +func (groupsBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, @@ -42,10 +50,10 @@ func ProduceBackupCollections( su support.StatusUpdater, counter *count.Bus, errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, error) { +) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { b, err := bpc.Selector.ToGroupsBackup() if err != nil { - return nil, nil, clues.Wrap(err, "groupsDataCollection: parsing selector") + return nil, nil, true, clues.Wrap(err, "groupsDataCollection: parsing selector") } var ( @@ -66,7 +74,7 @@ func ProduceBackupCollections( bpc.ProtectedResource.ID(), api.CallConfig{}) if err != nil { - return nil, nil, clues.WrapWC(ctx, err, "getting group") + return nil, nil, true, clues.WrapWC(ctx, err, "getting group") } bc := backupCommon{ac, bpc, creds, group, sitesPreviousPaths, su} @@ -129,7 +137,7 @@ func ProduceBackupCollections( counter, errs) if err != nil { - return nil, nil, err + return nil, nil, true, err } collections = append(collections, baseCols...) @@ -143,7 +151,7 @@ func ProduceBackupCollections( su, counter) if err != nil { - return nil, nil, err + return nil, nil, true, err } collections = append(collections, md) @@ -152,7 +160,7 @@ func ProduceBackupCollections( logger.Ctx(ctx).Infow("produced collections", "stats", counter.Values()) - return collections, globalItemIDExclusions.ToReader(), el.Failure() + return collections, globalItemIDExclusions.ToReader(), true, el.Failure() } type backupCommon struct { diff --git a/src/internal/m365/service/onedrive/backup.go b/src/internal/m365/service/onedrive/backup.go index 0c19020ed..ece8c21a6 100644 --- a/src/internal/m365/service/onedrive/backup.go +++ b/src/internal/m365/service/onedrive/backup.go @@ -22,7 +22,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type oneDriveBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *oneDriveBackup { + return &oneDriveBackup{} +} + +func (oneDriveBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/sharepoint/backup.go b/src/internal/m365/service/sharepoint/backup.go index 9cebb74a3..46706a180 100644 --- a/src/internal/m365/service/sharepoint/backup.go +++ b/src/internal/m365/service/sharepoint/backup.go @@ -20,7 +20,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type sharePointBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *sharePointBackup { + return &sharePointBackup{} +} + +func (sharePointBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/teamschats/backup.go b/src/internal/m365/service/teamschats/backup.go new file mode 100644 index 000000000..da17cc8f4 --- /dev/null +++ b/src/internal/m365/service/teamschats/backup.go @@ -0,0 +1,168 @@ +package teamschats + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/common/prefixmatcher" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/teamschats" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +type teamsChatsBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *teamsChatsBackup { + return &teamsChatsBackup{} +} + +func (teamsChatsBackup) ProduceBackupCollections( + ctx context.Context, + bpc inject.BackupProducerConfig, + ac api.Client, + creds account.M365Config, + su support.StatusUpdater, + counter *count.Bus, + errs *fault.Bus, +) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { + b, err := bpc.Selector.ToTeamsChatsBackup() + if err != nil { + return nil, nil, true, clues.WrapWC(ctx, err, "parsing selector") + } + + var ( + el = errs.Local() + collections = []data.BackupCollection{} + categories = map[path.CategoryType]struct{}{} + ) + + ctx = clues.Add( + ctx, + "user_id", clues.Hide(bpc.ProtectedResource.ID()), + "user_name", clues.Hide(bpc.ProtectedResource.Name())) + + bc := backupCommon{ + apiCli: ac, + producerConfig: bpc, + creds: creds, + user: bpc.ProtectedResource, + statusUpdater: su, + } + + for _, scope := range b.Scopes() { + if el.Failure() != nil { + break + } + + cl := counter.Local() + ictx := clues.AddLabelCounter(ctx, cl.PlainAdder()) + ictx = clues.Add(ictx, "category", scope.Category().PathType()) + + var colls []data.BackupCollection + + switch scope.Category().PathType() { + case path.ChatsCategory: + colls, err = backupChats( + ictx, + bc, + scope, + cl, + el) + } + + if err != nil { + el.AddRecoverable(ctx, clues.Stack(err)) + continue + } + + collections = append(collections, colls...) + + categories[scope.Category().PathType()] = struct{}{} + } + + if len(collections) > 0 { + baseCols, err := graph.BaseCollections( + ctx, + collections, + creds.AzureTenantID, + bpc.ProtectedResource.ID(), + path.TeamsChatsService, + categories, + su, + counter, + errs) + if err != nil { + return nil, nil, true, err + } + + collections = append(collections, baseCols...) + } + + counter.Add(count.Collections, int64(len(collections))) + + logger.Ctx(ctx).Infow("produced collections", "stats", counter.Values()) + + return collections, nil, true, clues.Stack(el.Failure()).OrNil() +} + +type backupCommon struct { + apiCli api.Client + producerConfig inject.BackupProducerConfig + creds account.M365Config + user idname.Provider + statusUpdater support.StatusUpdater +} + +func backupChats( + ctx context.Context, + bc backupCommon, + scope selectors.TeamsChatsScope, + counter *count.Bus, + errs *fault.Bus, +) ([]data.BackupCollection, error) { + var colls []data.BackupCollection + + progressMessage := observe.MessageWithCompletion( + ctx, + observe.ProgressCfg{ + Indent: 1, + CompletionMessage: func() string { return "(done)" }, + }, + scope.Category().PathType().HumanString()) + defer close(progressMessage) + + bh := teamschats.NewUsersChatsBackupHandler( + bc.creds.AzureTenantID, + bc.producerConfig.ProtectedResource.ID(), + bc.apiCli.Chats()) + + // Always disable lazy reader for channels until #4321 support is added + useLazyReader := false + + colls, _, err := teamschats.CreateCollections( + ctx, + bc.producerConfig, + bh, + bc.creds.AzureTenantID, + scope, + bc.statusUpdater, + useLazyReader, + counter, + errs) + + return colls, clues.Stack(err).OrNil() +} diff --git a/src/internal/m365/service/teamschats/enabled.go b/src/internal/m365/service/teamschats/enabled.go new file mode 100644 index 000000000..a4613feab --- /dev/null +++ b/src/internal/m365/service/teamschats/enabled.go @@ -0,0 +1,18 @@ +package teamschats + +import ( + "context" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +func IsServiceEnabled( + ctx context.Context, + gbi api.GetByIDer[models.Userable], + resource string, +) (bool, error) { + // TODO(rkeepers): investgate service enablement checks + return true, nil +} diff --git a/src/internal/m365/service/teamschats/enabled_test.go b/src/internal/m365/service/teamschats/enabled_test.go new file mode 100644 index 000000000..6c79549aa --- /dev/null +++ b/src/internal/m365/service/teamschats/enabled_test.go @@ -0,0 +1,69 @@ +package teamschats + +import ( + "context" + "testing" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type EnabledUnitSuite struct { + tester.Suite +} + +func TestEnabledUnitSuite(t *testing.T) { + suite.Run(t, &EnabledUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +var _ api.GetByIDer[models.Userable] = mockGU{} + +type mockGU struct { + user models.Userable + err error +} + +func (m mockGU) GetByID( + ctx context.Context, + identifier string, + _ api.CallConfig, +) (models.Userable, error) { + return m.user, m.err +} + +func (suite *EnabledUnitSuite) TestIsServiceEnabled() { + table := []struct { + name string + mock func(context.Context) api.GetByIDer[models.Userable] + expect assert.BoolAssertionFunc + expectErr assert.ErrorAssertionFunc + }{ + { + name: "ok", + mock: func(ctx context.Context) api.GetByIDer[models.Userable] { + return mockGU{} + }, + expect: assert.True, + expectErr: assert.NoError, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + gu := test.mock(ctx) + + ok, err := IsServiceEnabled(ctx, gu, "resource_id") + test.expect(t, ok, "has mailbox flag") + test.expectErr(t, err, clues.ToCore(err)) + }) + } +} diff --git a/src/internal/m365/service/teamschats/mock/mock.go b/src/internal/m365/service/teamschats/mock/mock.go new file mode 100644 index 000000000..5ba482412 --- /dev/null +++ b/src/internal/m365/service/teamschats/mock/mock.go @@ -0,0 +1,14 @@ +package stub + +import ( + "github.com/alcionai/corso/src/pkg/backup/details" +) + +func ItemInfo() details.ItemInfo { + return details.ItemInfo{ + TeamsChats: &details.TeamsChatsInfo{ + ItemType: details.TeamsChat, + Chat: details.ChatInfo{}, + }, + } +} diff --git a/src/pkg/count/keys.go b/src/pkg/count/keys.go index 9ee1fb9fc..87ce2ed75 100644 --- a/src/pkg/count/keys.go +++ b/src/pkg/count/keys.go @@ -42,6 +42,7 @@ const ( CollectionNotMoved Key = "collection-state-not-moved" CollectionTombstoned Key = "collection-state-tombstoned" Collections Key = "collections" + Containers Key = "containers" DeleteFolderMarker Key = "delete-folder-marker" DeleteItemMarker Key = "delete-item-marker" Drives Key = "drives" @@ -66,6 +67,7 @@ const ( Sites Key = "sites" Lists Key = "lists" SkippedContainers Key = "skipped-containers" + SkippedItems Key = "skipped-items" StreamBytesAdded Key = "stream-bytes-added" StreamDirsAdded Key = "stream-dirs-added" StreamDirsFound Key = "stream-dirs-found" diff --git a/src/pkg/path/service_category_test.go b/src/pkg/path/service_category_test.go index ce58d61c3..21b1684cb 100644 --- a/src/pkg/path/service_category_test.go +++ b/src/pkg/path/service_category_test.go @@ -163,9 +163,10 @@ func (suite *ServiceCategoryUnitSuite) TestToServiceType() { } for _, test := range table { suite.Run(test.name, func() { - t := suite.T() - - assert.Equal(t, test.expected, ToServiceType(test.service)) + assert.Equal( + suite.T(), + test.expected.String(), + ToServiceType(test.service).String()) }) } } @@ -189,9 +190,10 @@ func (suite *ServiceCategoryUnitSuite) TestToCategoryType() { } for _, test := range table { suite.Run(test.name, func() { - t := suite.T() - - assert.Equal(t, test.expected, ToCategoryType(test.category)) + assert.Equal( + suite.T(), + test.expected.String(), + ToCategoryType(test.category).String()) }) } } diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 4c49f637b..03654d8d1 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -355,6 +355,9 @@ func selectorAsIface[T any](s Selector) (T, error) { case ServiceGroups: a, err = func() (any, error) { return s.ToGroupsRestore() }() t = a.(T) + case ServiceTeamsChats: + a, err = func() (any, error) { return s.ToTeamsChatsRestore() }() + t = a.(T) default: err = clues.Stack(ErrorUnrecognizedService, clues.New(s.Service.String())) } diff --git a/src/pkg/selectors/testdata/groups.go b/src/pkg/selectors/testdata/groups.go index 5d928c2a1..39f316a6f 100644 --- a/src/pkg/selectors/testdata/groups.go +++ b/src/pkg/selectors/testdata/groups.go @@ -4,7 +4,10 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" ) -const TestChannelName = "Test" +const ( + TestChannelName = "Test" + TestChatTopic = "Test" +) // GroupsBackupFolderScope is the standard folder scope that should be used // in integration backups with groups when interacting with libraries. diff --git a/src/pkg/selectors/testdata/teamschats.go b/src/pkg/selectors/testdata/teamschats.go new file mode 100644 index 000000000..9786a1d50 --- /dev/null +++ b/src/pkg/selectors/testdata/teamschats.go @@ -0,0 +1,9 @@ +package testdata + +import "github.com/alcionai/corso/src/pkg/selectors" + +// TeamsChatsBackupChatScope is the standard folder scope that should be used +// in integration backups with teams chats when interacting with chats. +func TeamsChatsBackupChatScope(sel *selectors.TeamsChatsBackup) []selectors.TeamsChatsScope { + return sel.Chats([]string{TestChatTopic}) +} diff --git a/src/pkg/services/m365/api/graph/interfaces.go b/src/pkg/services/m365/api/graph/interfaces.go index 875ae652e..a498c59c6 100644 --- a/src/pkg/services/m365/api/graph/interfaces.go +++ b/src/pkg/services/m365/api/graph/interfaces.go @@ -12,6 +12,10 @@ type GetLastModifiedDateTimer interface { GetLastModifiedDateTime() *time.Time } +type GetLastUpdatedDateTimer interface { + GetLastUpdatedDateTime() *time.Time +} + type GetAdditionalDataer interface { GetAdditionalData() map[string]any }