From 8d3fdeeb8dbd8dc68284c1d33b0b431d309b4f5b Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 18 Aug 2023 09:09:08 -0700 Subject: [PATCH 01/16] Remove call to PITR backup check (#4067) Currently failing due to minor upstream bugs. Disable until we can get upstream fixes in. Revert this merge once upstream issues are fixed --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #4031 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/longevity_test/longevity.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index b3d6f865d..efe3bd352 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -67,6 +67,9 @@ func deleteBackups( // pitrListBackups connects to the repository at the given point in time and // lists the backups for service. It then checks the list of backups contains // the backups in backupIDs. +// +//nolint:unused +//lint:ignore U1000 Waiting for upstream fix tracked by 4031 func pitrListBackups( ctx context.Context, service path.ServiceType, @@ -156,16 +159,10 @@ func main() { fatal(ctx, "invalid number of days provided", nil) } - beforeDel := time.Now() - - backups, err := deleteBackups(ctx, service, days) + _, err = deleteBackups(ctx, service, days) if err != nil { fatal(ctx, "deleting backups", clues.Stack(err)) } - - if err := pitrListBackups(ctx, service, beforeDel, backups); err != nil { - fatal(ctx, "listing backups from point in time", clues.Stack(err)) - } } func fatal(ctx context.Context, msg string, err error) { From 20675dbcf7086f24ff086b593b3836519779b1b6 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 18 Aug 2023 14:10:56 -0600 Subject: [PATCH 02/16] add the groups resources service addition (#4053) Adds groups to the m365 services api. Also adds a bit of touchups/cleanups on the side. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3989 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../tester/tconfig/protected_resources.go | 4 +- src/pkg/services/m365/api/groups.go | 106 ++++++----------- src/pkg/services/m365/api/groups_test.go | 93 +-------------- src/pkg/services/m365/api/helper_test.go | 13 ++- src/pkg/services/m365/groups.go | 97 ++++++++++++++++ src/pkg/services/m365/groups_test.go | 108 ++++++++++++++++++ src/pkg/services/m365/m365.go | 15 ++- src/pkg/services/m365/m365_test.go | 18 +-- 8 files changed, 272 insertions(+), 182 deletions(-) create mode 100644 src/pkg/services/m365/groups.go create mode 100644 src/pkg/services/m365/groups_test.go diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index caac0c586..26c0187ac 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -223,11 +223,11 @@ func UnlicensedM365UserID(t *testing.T) string { // Teams -// M365TeamsID returns a teamID string representing the m365TeamsID described +// M365TeamID returns a teamID string representing the m365TeamsID described // by either the env var CORSO_M365_TEST_TEAM_ID, the corso_test.toml config // file or the default value (in that order of priority). The default is a // last-attempt fallback that will only work on alcion's testing org. -func M365TeamsID(t *testing.T) string { +func M365TeamID(t *testing.T) string { cfg, err := ReadTestConfig() require.NoError(t, err, "retrieving m365 team id from test configuration: %+v", clues.ToCore(err)) diff --git a/src/pkg/services/m365/api/groups.go b/src/pkg/services/m365/api/groups.go index c2a27dad3..3d036e610 100644 --- a/src/pkg/services/m365/api/groups.go +++ b/src/pkg/services/m365/api/groups.go @@ -49,24 +49,6 @@ func (c Groups) GetAll( return getGroups(ctx, errs, service) } -// GetTeams retrieves all Teams. -func (c Groups) GetTeams( - ctx context.Context, - errs *fault.Bus, -) ([]models.Groupable, error) { - service, err := c.Service() - if err != nil { - return nil, err - } - - groups, err := getGroups(ctx, errs, service) - if err != nil { - return nil, err - } - - return OnlyTeams(ctx, groups), nil -} - // GetAll retrieves all groups. func getGroups( ctx context.Context, @@ -113,31 +95,6 @@ func getGroups( return groups, el.Failure() } -func OnlyTeams(ctx context.Context, groups []models.Groupable) []models.Groupable { - log := logger.Ctx(ctx) - - var teams []models.Groupable - - for _, g := range groups { - if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { - val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, g.GetAdditionalData()) - for _, v := range val { - s, err := str.AnyToString(v) - if err != nil { - log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) - continue - } - - if s == teamsAdditionalDataLabel { - teams = append(teams, g) - } - } - } - } - - return teams -} - // GetID retrieves group by groupID. func (c Groups) GetByID( ctx context.Context, @@ -158,34 +115,6 @@ func (c Groups) GetByID( return resp, graph.Stack(ctx, err).OrNil() } -// GetTeamByID retrieves group by groupID. -func (c Groups) GetTeamByID( - ctx context.Context, - identifier string, -) (models.Groupable, error) { - service, err := c.Service() - if err != nil { - return nil, err - } - - resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) - if err != nil { - err := graph.Wrap(ctx, err, "getting group by id") - - return nil, err - } - - groups := []models.Groupable{resp} - - if len(OnlyTeams(ctx, groups)) == 0 { - err := clues.New("given teamID is not related to any team") - - return nil, err - } - - return resp, graph.Stack(ctx, err).OrNil() -} - // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- @@ -203,3 +132,38 @@ func ValidateGroup(item models.Groupable) error { return nil } + +func OnlyTeams(ctx context.Context, groups []models.Groupable) []models.Groupable { + var teams []models.Groupable + + for _, g := range groups { + if IsTeam(ctx, g) { + teams = append(teams, g) + } + } + + return teams +} + +func IsTeam(ctx context.Context, mg models.Groupable) bool { + log := logger.Ctx(ctx) + + if mg.GetAdditionalData()[ResourceProvisioningOptions] == nil { + return false + } + + val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, mg.GetAdditionalData()) + for _, v := range val { + s, err := str.AnyToString(v) + if err != nil { + log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) + continue + } + + if s == teamsAdditionalDataLabel { + return true + } + } + + return false +} diff --git a/src/pkg/services/m365/api/groups_test.go b/src/pkg/services/m365/api/groups_test.go index 8ce0f8f6b..ae435168a 100644 --- a/src/pkg/services/m365/api/groups_test.go +++ b/src/pkg/services/m365/api/groups_test.go @@ -97,7 +97,7 @@ func (suite *GroupsIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *GroupsIntgSuite) TestGetAllGroups() { +func (suite *GroupsIntgSuite) TestGetAll() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -107,100 +107,15 @@ func (suite *GroupsIntgSuite) TestGetAllGroups() { Groups(). GetAll(ctx, fault.New(true)) require.NoError(t, err) - require.NotZero(t, len(groups), "must have at least one group") -} - -func (suite *GroupsIntgSuite) TestGetAllTeams() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - teams, err := suite.its.ac. - Groups(). - GetTeams(ctx, fault.New(true)) - require.NoError(t, err) - require.NotZero(t, len(teams), "must have at least one teams") - - groups, err := suite.its.ac. - Groups(). - GetAll(ctx, fault.New(true)) - require.NoError(t, err) - require.NotZero(t, len(groups), "must have at least one group") - - var isTeam bool - - if len(groups) > len(teams) { - isTeam = true - } - - assert.True(t, isTeam, "must only return teams") -} - -func (suite *GroupsIntgSuite) TestTeams_GetByID() { - var ( - t = suite.T() - teamID = tconfig.M365TeamsID(t) - ) - - teamsAPI := suite.its.ac.Groups() - - table := []struct { - name string - id string - expectErr func(*testing.T, error) - }{ - { - name: "3 part id", - id: teamID, - expectErr: func(t *testing.T, err error) { - assert.NoError(t, err, clues.ToCore(err)) - }, - }, - { - name: "malformed id", - id: uuid.NewString(), - expectErr: func(t *testing.T, err error) { - assert.Error(t, err, clues.ToCore(err)) - }, - }, - { - name: "random id", - id: uuid.NewString() + "," + uuid.NewString(), - expectErr: func(t *testing.T, err error) { - assert.Error(t, err, clues.ToCore(err)) - }, - }, - - { - name: "malformed url", - id: "barunihlda", - expectErr: func(t *testing.T, err error) { - assert.Error(t, err, clues.ToCore(err)) - }, - }, - } - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - _, err := teamsAPI.GetTeamByID(ctx, test.id) - test.expectErr(t, err) - }) - } + require.NotZero(t, len(groups), "must find at least one group") } func (suite *GroupsIntgSuite) TestGroups_GetByID() { var ( - t = suite.T() - groupID = tconfig.M365GroupID(t) + groupID = suite.its.groupID + groupsAPI = suite.its.ac.Groups() ) - groupsAPI := suite.its.ac.Groups() - table := []struct { name string id string diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index a9c12324f..8e8c760c0 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -83,7 +83,7 @@ type intgTesterSetup struct { siteID string siteDriveID string siteDriveRootFolderID string - teamID string + groupID string } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -132,13 +132,16 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) - // teams - its.teamID = tconfig.M365TeamsID(t) + // group - team, err := its.ac.Groups().GetTeamByID(ctx, its.teamID) + // use of the TeamID is intentional here, so that we are assured + // the group has full usage of the teams api. + its.groupID = tconfig.M365TeamID(t) + + team, err := its.ac.Groups().GetByID(ctx, its.groupID) require.NoError(t, err, clues.ToCore(err)) - its.teamID = ptr.Val(team.GetId()) + its.groupID = ptr.Val(team.GetId()) return its } diff --git a/src/pkg/services/m365/groups.go b/src/pkg/services/m365/groups.go new file mode 100644 index 000000000..f4924be22 --- /dev/null +++ b/src/pkg/services/m365/groups.go @@ -0,0 +1,97 @@ +package m365 + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +// Group is the minimal information required to identify and display a M365 Group. +type Group struct { + ID string + + // DisplayName is the human-readable name of the group. Normally the plaintext name that the + // user provided when they created the group, or the updated name if it was changed. + // Ex: displayName: "My Group" + DisplayName string + + // IsTeam is true if the group qualifies as a Teams resource, and is able to backup and restore + // teams data. + IsTeam bool +} + +// GroupsCompat returns a list of groups in the specified M365 tenant. +func GroupsCompat(ctx context.Context, acct account.Account) ([]*Group, error) { + errs := fault.New(true) + + us, err := Groups(ctx, acct, errs) + if err != nil { + return nil, err + } + + return us, errs.Failure() +} + +// Groups returns a list of groups in the specified M365 tenant +func Groups( + ctx context.Context, + acct account.Account, + errs *fault.Bus, +) ([]*Group, error) { + ac, err := makeAC(ctx, acct, path.GroupsService) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx) + } + + return getAllGroups(ctx, ac.Groups()) +} + +func getAllGroups( + ctx context.Context, + ga getAller[models.Groupable], +) ([]*Group, error) { + groups, err := ga.GetAll(ctx, fault.New(true)) + if err != nil { + return nil, clues.Wrap(err, "retrieving groups") + } + + ret := make([]*Group, 0, len(groups)) + + for _, g := range groups { + t, err := parseGroup(ctx, g) + if err != nil { + return nil, clues.Wrap(err, "parsing groups") + } + + ret = append(ret, t) + } + + return ret, nil +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// parseUser extracts information from `models.Groupable` we care about +func parseGroup(ctx context.Context, mg models.Groupable) (*Group, error) { + if mg.GetDisplayName() == nil { + return nil, clues.New("group missing display name"). + With("group_id", ptr.Val(mg.GetId())) + } + + u := &Group{ + ID: ptr.Val(mg.GetId()), + DisplayName: ptr.Val(mg.GetDisplayName()), + IsTeam: api.IsTeam(ctx, mg), + } + + return u, nil +} diff --git a/src/pkg/services/m365/groups_test.go b/src/pkg/services/m365/groups_test.go new file mode 100644 index 000000000..8fa650a98 --- /dev/null +++ b/src/pkg/services/m365/groups_test.go @@ -0,0 +1,108 @@ +package m365_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/credentials" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365" +) + +type GroupsIntgSuite struct { + tester.Suite + acct account.Account +} + +func TestGroupsIntgSuite(t *testing.T) { + suite.Run(t, &GroupsIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *GroupsIntgSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, true, 4) + + suite.acct = tconfig.NewM365Account(t) +} + +func (suite *GroupsIntgSuite) TestGroups() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, true, 4) + + groups, err := m365.Groups(ctx, suite.acct, fault.New(true)) + assert.NoError(t, err, clues.ToCore(err)) + assert.NotEmpty(t, groups) + + for _, group := range groups { + suite.Run("group_"+group.ID, func() { + t := suite.T() + + assert.NotEmpty(t, group.ID) + assert.NotEmpty(t, group.DisplayName) + + // at least one known group should be a team + if group.ID == tconfig.M365TeamID(t) { + assert.True(t, group.IsTeam) + } + }) + } +} + +func (suite *GroupsIntgSuite) TestGroups_InvalidCredentials() { + table := []struct { + name string + acct func(t *testing.T) account.Account + }{ + { + name: "Invalid Credentials", + acct: func(t *testing.T) account.Account { + a, err := account.NewAccount( + account.ProviderM365, + account.M365Config{ + M365: credentials.M365{ + AzureClientID: "Test", + AzureClientSecret: "without", + }, + AzureTenantID: "data", + }, + ) + require.NoError(t, err, clues.ToCore(err)) + + return a + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + groups, err := m365.Groups(ctx, test.acct(t), fault.New(true)) + assert.Empty(t, groups, "returned no groups") + assert.NotNil(t, err) + }) + } +} diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 5b61885e5..469f4d08f 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -24,6 +24,10 @@ type getDefaultDriver interface { GetDefaultDrive(ctx context.Context, userID string) (models.Driveable, error) } +type getAller[T any] interface { + GetAll(ctx context.Context, errs *fault.Bus) ([]T, error) +} + // --------------------------------------------------------------------------- // Users // --------------------------------------------------------------------------- @@ -253,12 +257,11 @@ func Sites(ctx context.Context, acct account.Account, errs *fault.Bus) ([]*Site, return getAllSites(ctx, ac.Sites()) } -type getAllSiteser interface { - GetAll(ctx context.Context, errs *fault.Bus) ([]models.Siteable, error) -} - -func getAllSites(ctx context.Context, gas getAllSiteser) ([]*Site, error) { - sites, err := gas.GetAll(ctx, fault.New(true)) +func getAllSites( + ctx context.Context, + ga getAller[models.Siteable], +) ([]*Site, error) { + sites, err := ga.GetAll(ctx, fault.New(true)) if err != nil { if clues.HasLabel(err, graph.LabelsNoSharePointLicense) { return nil, clues.Stack(graph.ErrServiceNotEnabled, err) diff --git a/src/pkg/services/m365/m365_test.go b/src/pkg/services/m365/m365_test.go index 1eafa67f2..0124f13f2 100644 --- a/src/pkg/services/m365/m365_test.go +++ b/src/pkg/services/m365/m365_test.go @@ -276,25 +276,25 @@ func (suite *m365UnitSuite) TestCheckUserHasDrives() { } } -type mockGAS struct { +type mockGASites struct { response []models.Siteable err error } -func (m mockGAS) GetAll(context.Context, *fault.Bus) ([]models.Siteable, error) { +func (m mockGASites) GetAll(context.Context, *fault.Bus) ([]models.Siteable, error) { return m.response, m.err } func (suite *m365UnitSuite) TestGetAllSites() { table := []struct { name string - mock func(context.Context) getAllSiteser + mock func(context.Context) getAller[models.Siteable] expectErr func(*testing.T, error) }{ { name: "ok", - mock: func(ctx context.Context) getAllSiteser { - return mockGAS{[]models.Siteable{}, nil} + mock: func(ctx context.Context) getAller[models.Siteable] { + return mockGASites{[]models.Siteable{}, nil} }, expectErr: func(t *testing.T, err error) { assert.NoError(t, err, clues.ToCore(err)) @@ -302,14 +302,14 @@ func (suite *m365UnitSuite) TestGetAllSites() { }, { name: "no sharepoint license", - mock: func(ctx context.Context) getAllSiteser { + mock: func(ctx context.Context) getAller[models.Siteable] { odErr := odataerrors.NewODataError() merr := odataerrors.NewMainError() merr.SetCode(ptr.To("code")) merr.SetMessage(ptr.To(string(graph.NoSPLicense))) odErr.SetErrorEscaped(merr) - return mockGAS{nil, graph.Stack(ctx, odErr)} + return mockGASites{nil, graph.Stack(ctx, odErr)} }, expectErr: func(t *testing.T, err error) { assert.ErrorIs(t, err, graph.ErrServiceNotEnabled, clues.ToCore(err)) @@ -317,14 +317,14 @@ func (suite *m365UnitSuite) TestGetAllSites() { }, { name: "arbitrary error", - mock: func(ctx context.Context) getAllSiteser { + mock: func(ctx context.Context) getAller[models.Siteable] { odErr := odataerrors.NewODataError() merr := odataerrors.NewMainError() merr.SetCode(ptr.To("code")) merr.SetMessage(ptr.To("message")) odErr.SetErrorEscaped(merr) - return mockGAS{nil, graph.Stack(ctx, odErr)} + return mockGASites{nil, graph.Stack(ctx, odErr)} }, expectErr: func(t *testing.T, err error) { assert.Error(t, err, clues.ToCore(err)) From 2c00ca40ac76e5135c4fd53cda70c6ec0d72b57a Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 18 Aug 2023 15:35:22 -0600 Subject: [PATCH 03/16] Updated teams cli addition (#4054) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3989 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- src/cli/backup/backup.go | 1 + src/cli/backup/groups.go | 230 +++++++++++++++++++++++++++++++++ src/cli/backup/groups_test.go | 98 ++++++++++++++ src/cli/backup/teams.go | 230 +++++++++++++++++++++++++++++++++ src/cli/backup/teams_test.go | 98 ++++++++++++++ src/cli/flags/groups.go | 28 ++++ src/cli/flags/teams.go | 28 ++++ src/cli/restore/groups.go | 81 ++++++++++++ src/cli/restore/groups_test.go | 108 ++++++++++++++++ src/cli/restore/teams.go | 81 ++++++++++++ src/cli/restore/teams_test.go | 108 ++++++++++++++++ src/cli/utils/groups.go | 30 +++++ src/cli/utils/teams.go | 30 +++++ src/cli/utils/utils.go | 2 + src/pkg/path/service_type.go | 2 + 15 files changed, 1155 insertions(+) create mode 100644 src/cli/backup/groups.go create mode 100644 src/cli/backup/groups_test.go create mode 100644 src/cli/backup/teams.go create mode 100644 src/cli/backup/teams_test.go create mode 100644 src/cli/flags/groups.go create mode 100644 src/cli/flags/teams.go create mode 100644 src/cli/restore/groups.go create mode 100644 src/cli/restore/groups_test.go create mode 100644 src/cli/restore/teams.go create mode 100644 src/cli/restore/teams_test.go create mode 100644 src/cli/utils/groups.go create mode 100644 src/cli/utils/teams.go diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index c8df39902..56b5c5ef4 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -39,6 +39,7 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{ addExchangeCommands, addOneDriveCommands, addSharePointCommands, + addTeamsCommands, } // AddCommands attaches all `corso backup * *` commands to the parent. diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go new file mode 100644 index 000000000..3f1f83eb7 --- /dev/null +++ b/src/cli/backup/groups.go @@ -0,0 +1,230 @@ +package backup + +import ( + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/pkg/path" +) + +// ------------------------------------------------------------------------------------------------ +// setup and globals +// ------------------------------------------------------------------------------------------------ + +const ( + groupsServiceCommand = "groups" + groupsServiceCommandCreateUseSuffix = "--group | '" + flags.Wildcard + "'" + groupsServiceCommandDeleteUseSuffix = "--backup " + groupsServiceCommandDetailsUseSuffix = "--backup " +) + +// TODO: correct examples +const ( + groupsServiceCommandCreateExamples = `# Backup all Groups data for Alice +corso backup create groups --group alice@example.com + +# Backup only Groups contacts for Alice and Bob +corso backup create groups --group engineering,sales --data contacts + +# Backup all Groups data for all M365 users +corso backup create groups --group '*'` + + groupsServiceCommandDeleteExamples = `# Delete Groups backup with ID 1234abcd-12ab-cd34-56de-1234abcd +corso backup delete groups --backup 1234abcd-12ab-cd34-56de-1234abcd` + + groupsServiceCommandDetailsExamples = `# Explore items in Alice's latest backup (1234abcd...) +corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd + +# Explore calendar events occurring after start of 2022 +corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --event-starts-after 2022-01-01T00:00:00` +) + +// called by backup.go to map subcommands to provider-specific handling. +func addGroupsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case createCommand: + c, fs = utils.AddCommand(cmd, groupsCreateCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + groupsServiceCommandCreateUseSuffix + c.Example = groupsServiceCommandCreateExamples + + // Flags addition ordering should follow the order we want them to appear in help and docs: + flags.AddGroupFlag(c) + flags.AddDataFlag(c, []string{dataLibraries}, false) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + flags.AddFetchParallelismFlag(c) + flags.AddFailFastFlag(c) + + case listCommand: + c, fs = utils.AddCommand(cmd, groupsListCmd(), utils.HideCommand()) + fs.SortFlags = false + + flags.AddBackupIDFlag(c, false) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + addFailedItemsFN(c) + addSkippedItemsFN(c) + addRecoveredErrorsFN(c) + + case detailsCommand: + c, fs = utils.AddCommand(cmd, groupsDetailsCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + groupsServiceCommandDetailsUseSuffix + c.Example = groupsServiceCommandDetailsExamples + + flags.AddSkipReduceFlag(c) + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + flags.AddBackupIDFlag(c, true) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + + case deleteCommand: + c, fs = utils.AddCommand(cmd, groupsDeleteCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + groupsServiceCommandDeleteUseSuffix + c.Example = groupsServiceCommandDeleteExamples + + flags.AddBackupIDFlag(c, true) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + } + + return c +} + +// ------------------------------------------------------------------------------------------------ +// backup create +// ------------------------------------------------------------------------------------------------ + +// `corso backup create groups [...]` +func groupsCreateCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "Backup M365 Group service data", + RunE: createGroupsCmd, + Args: cobra.NoArgs, + } +} + +// processes a groups service backup. +func createGroupsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} + +// ------------------------------------------------------------------------------------------------ +// backup list +// ------------------------------------------------------------------------------------------------ + +// `corso backup list groups [...]` +func groupsListCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "List the history of M365 Groups service backups", + RunE: listGroupsCmd, + Args: cobra.NoArgs, + } +} + +// lists the history of backup operations +func listGroupsCmd(cmd *cobra.Command, args []string) error { + return genericListCommand(cmd, flags.BackupIDFV, path.GroupsService, args) +} + +// ------------------------------------------------------------------------------------------------ +// backup details +// ------------------------------------------------------------------------------------------------ + +// `corso backup details groups [...]` +func groupsDetailsCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "Shows the details of a M365 Groups service backup", + RunE: detailsGroupsCmd, + Args: cobra.NoArgs, + } +} + +// processes a groups service backup. +func detailsGroupsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if err := validateGroupBackupCreateFlags(flags.GroupFV); err != nil { + return Only(ctx, err) + } + + return Only(ctx, utils.ErrNotYetImplemented) +} + +// ------------------------------------------------------------------------------------------------ +// backup delete +// ------------------------------------------------------------------------------------------------ + +// `corso backup delete groups [...]` +func groupsDeleteCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "Delete backed-up M365 Groups service data", + RunE: deleteGroupsCmd, + Args: cobra.NoArgs, + } +} + +// deletes an groups service backup. +func deleteGroupsCmd(cmd *cobra.Command, args []string) error { + return genericDeleteCommand(cmd, path.GroupsService, flags.BackupIDFV, "Groups", args) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func validateGroupBackupCreateFlags(groups []string) error { + if len(groups) == 0 { + return clues.New( + "requires one or more --" + + flags.GroupFN + " ids, or the wildcard --" + + flags.GroupFN + " *", + ) + } + + // TODO(meain) + // for _, d := range cats { + // if d != dataLibraries { + // return clues.New( + // d + " is an unrecognized data type; only " + dataLibraries + " is supported" + // ) + // } + // } + + return nil +} diff --git a/src/cli/backup/groups_test.go b/src/cli/backup/groups_test.go new file mode 100644 index 000000000..04a131b59 --- /dev/null +++ b/src/cli/backup/groups_test.go @@ -0,0 +1,98 @@ +package backup + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/tester" +) + +type GroupsUnitSuite struct { + tester.Suite +} + +func TestGroupsUnitSuite(t *testing.T) { + suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *GroupsUnitSuite) TestAddGroupsCommands() { + expectUse := groupsServiceCommand + + table := []struct { + name string + use string + expectUse string + expectShort string + flags []string + expectRunE func(*cobra.Command, []string) error + }{ + { + "create groups", + createCommand, + expectUse + " " + groupsServiceCommandCreateUseSuffix, + groupsCreateCmd().Short, + []string{ + flags.CategoryDataFN, + flags.FailFastFN, + flags.FetchParallelismFN, + flags.SkipReduceFN, + flags.NoStatsFN, + }, + createGroupsCmd, + }, + { + "list groups", + listCommand, + expectUse, + groupsListCmd().Short, + []string{ + flags.BackupFN, + flags.FailedItemsFN, + flags.SkippedItemsFN, + flags.RecoveredErrorsFN, + }, + listGroupsCmd, + }, + { + "details groups", + detailsCommand, + expectUse + " " + groupsServiceCommandDetailsUseSuffix, + groupsDetailsCmd().Short, + []string{ + flags.BackupFN, + }, + detailsGroupsCmd, + }, + { + "delete groups", + deleteCommand, + expectUse + " " + groupsServiceCommandDeleteUseSuffix, + groupsDeleteCmd().Short, + []string{flags.BackupFN}, + deleteGroupsCmd, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + c := addGroupsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + }) + } +} diff --git a/src/cli/backup/teams.go b/src/cli/backup/teams.go new file mode 100644 index 000000000..fcac3394d --- /dev/null +++ b/src/cli/backup/teams.go @@ -0,0 +1,230 @@ +package backup + +import ( + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/pkg/path" +) + +// ------------------------------------------------------------------------------------------------ +// setup and globals +// ------------------------------------------------------------------------------------------------ + +const ( + teamsServiceCommand = "teams" + teamsServiceCommandCreateUseSuffix = "--team | '" + flags.Wildcard + "'" + teamsServiceCommandDeleteUseSuffix = "--backup " + teamsServiceCommandDetailsUseSuffix = "--backup " +) + +// TODO: correct examples +const ( + teamsServiceCommandCreateExamples = `# Backup all Teams data for Alice +corso backup create teams --team alice@example.com + +# Backup only Teams contacts for Alice and Bob +corso backup create teams --team engineering,sales --data contacts + +# Backup all Teams data for all M365 users +corso backup create teams --team '*'` + + teamsServiceCommandDeleteExamples = `# Delete Teams backup with ID 1234abcd-12ab-cd34-56de-1234abcd +corso backup delete teams --backup 1234abcd-12ab-cd34-56de-1234abcd` + + teamsServiceCommandDetailsExamples = `# Explore items in Alice's latest backup (1234abcd...) +corso backup details teams --backup 1234abcd-12ab-cd34-56de-1234abcd + +# Explore calendar events occurring after start of 2022 +corso backup details teams --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --event-starts-after 2022-01-01T00:00:00` +) + +// called by backup.go to map subcommands to provider-specific handling. +func addTeamsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case createCommand: + c, fs = utils.AddCommand(cmd, teamsCreateCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + teamsServiceCommandCreateUseSuffix + c.Example = teamsServiceCommandCreateExamples + + // Flags addition ordering should follow the order we want them to appear in help and docs: + flags.AddTeamFlag(c) + flags.AddDataFlag(c, []string{dataEmail, dataContacts, dataEvents}, false) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + flags.AddFetchParallelismFlag(c) + flags.AddFailFastFlag(c) + + case listCommand: + c, fs = utils.AddCommand(cmd, teamsListCmd(), utils.HideCommand()) + fs.SortFlags = false + + flags.AddBackupIDFlag(c, false) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + addFailedItemsFN(c) + addSkippedItemsFN(c) + addRecoveredErrorsFN(c) + + case detailsCommand: + c, fs = utils.AddCommand(cmd, teamsDetailsCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + teamsServiceCommandDetailsUseSuffix + c.Example = teamsServiceCommandDetailsExamples + + flags.AddSkipReduceFlag(c) + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + flags.AddBackupIDFlag(c, true) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + + case deleteCommand: + c, fs = utils.AddCommand(cmd, teamsDeleteCmd(), utils.HideCommand()) + fs.SortFlags = false + + c.Use = c.Use + " " + teamsServiceCommandDeleteUseSuffix + c.Example = teamsServiceCommandDeleteExamples + + flags.AddBackupIDFlag(c, true) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + } + + return c +} + +// ------------------------------------------------------------------------------------------------ +// backup create +// ------------------------------------------------------------------------------------------------ + +// `corso backup create teams [...]` +func teamsCreateCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "Backup M365 Team service data", + RunE: createTeamsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teams service backup. +func createTeamsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if err := validateTeamBackupCreateFlags(flags.TeamFV); err != nil { + return Only(ctx, err) + } + + return Only(ctx, utils.ErrNotYetImplemented) +} + +// ------------------------------------------------------------------------------------------------ +// backup list +// ------------------------------------------------------------------------------------------------ + +// `corso backup list teams [...]` +func teamsListCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "List the history of M365 Teams service backups", + RunE: listTeamsCmd, + Args: cobra.NoArgs, + } +} + +// lists the history of backup operations +func listTeamsCmd(cmd *cobra.Command, args []string) error { + return genericListCommand(cmd, flags.BackupIDFV, path.TeamsService, args) +} + +// ------------------------------------------------------------------------------------------------ +// backup details +// ------------------------------------------------------------------------------------------------ + +// `corso backup details teams [...]` +func teamsDetailsCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "Shows the details of a M365 Teams service backup", + RunE: detailsTeamsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teams service backup. +func detailsTeamsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} + +// ------------------------------------------------------------------------------------------------ +// backup delete +// ------------------------------------------------------------------------------------------------ + +// `corso backup delete teams [...]` +func teamsDeleteCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "Delete backed-up M365 Teams service data", + RunE: deleteTeamsCmd, + Args: cobra.NoArgs, + } +} + +// deletes an teams service backup. +func deleteTeamsCmd(cmd *cobra.Command, args []string) error { + return genericDeleteCommand(cmd, path.TeamsService, flags.BackupIDFV, "Teams", args) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func validateTeamBackupCreateFlags(teams []string) error { + if len(teams) == 0 { + return clues.New( + "requires one or more --" + + flags.TeamFN + " ids, or the wildcard --" + + flags.TeamFN + " *", + ) + } + + // TODO(meain) + // for _, d := range cats { + // if d != dataLibraries { + // return clues.New( + // d + " is an unrecognized data type; only " + dataLibraries + " is supported" + // ) + // } + // } + + return nil +} diff --git a/src/cli/backup/teams_test.go b/src/cli/backup/teams_test.go new file mode 100644 index 000000000..966830f82 --- /dev/null +++ b/src/cli/backup/teams_test.go @@ -0,0 +1,98 @@ +package backup + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/tester" +) + +type TeamsUnitSuite struct { + tester.Suite +} + +func TestTeamsUnitSuite(t *testing.T) { + suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *TeamsUnitSuite) TestAddTeamsCommands() { + expectUse := teamsServiceCommand + + table := []struct { + name string + use string + expectUse string + expectShort string + flags []string + expectRunE func(*cobra.Command, []string) error + }{ + { + "create teams", + createCommand, + expectUse + " " + teamsServiceCommandCreateUseSuffix, + teamsCreateCmd().Short, + []string{ + flags.CategoryDataFN, + flags.FailFastFN, + flags.FetchParallelismFN, + flags.SkipReduceFN, + flags.NoStatsFN, + }, + createTeamsCmd, + }, + { + "list teams", + listCommand, + expectUse, + teamsListCmd().Short, + []string{ + flags.BackupFN, + flags.FailedItemsFN, + flags.SkippedItemsFN, + flags.RecoveredErrorsFN, + }, + listTeamsCmd, + }, + { + "details teams", + detailsCommand, + expectUse + " " + teamsServiceCommandDetailsUseSuffix, + teamsDetailsCmd().Short, + []string{ + flags.BackupFN, + }, + detailsTeamsCmd, + }, + { + "delete teams", + deleteCommand, + expectUse + " " + teamsServiceCommandDeleteUseSuffix, + teamsDeleteCmd().Short, + []string{flags.BackupFN}, + deleteTeamsCmd, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + c := addTeamsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + }) + } +} diff --git a/src/cli/flags/groups.go b/src/cli/flags/groups.go new file mode 100644 index 000000000..8aa6792ad --- /dev/null +++ b/src/cli/flags/groups.go @@ -0,0 +1,28 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ( + GroupFN = "group" +) + +var GroupFV []string + +func AddGroupDetailsAndRestoreFlags(cmd *cobra.Command) { + // TODO: implement flags +} + +// AddGroupFlag adds the --group flag, which accepts id or name values. +// TODO: need to decide what the appropriate "name" to accept here is. +// keepers thinks its either DisplayName or MailNickname or Mail +// Mail is most accurate, MailNickame is accurate and shorter, but the end user +// may not see either one visibly. +// https://learn.microsoft.com/en-us/graph/api/group-list?view=graph-rest-1.0&tabs=http +func AddGroupFlag(cmd *cobra.Command) { + cmd.Flags().StringSliceVar( + &GroupFV, + GroupFN, nil, + "Backup data by group; accepts '"+Wildcard+"' to select all groups.") +} diff --git a/src/cli/flags/teams.go b/src/cli/flags/teams.go new file mode 100644 index 000000000..a3ca73e62 --- /dev/null +++ b/src/cli/flags/teams.go @@ -0,0 +1,28 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ( + TeamFN = "team" +) + +var TeamFV []string + +func AddTeamDetailsAndRestoreFlags(cmd *cobra.Command) { + // TODO: implement flags +} + +// AddTeamFlag adds the --team flag, which accepts id or name values. +// TODO: need to decide what the appropriate "name" to accept here is. +// keepers thinks its either DisplayName or MailNickname or Mail +// Mail is most accurate, MailNickame is accurate and shorter, but the end user +// may not see either one visibly. +// https://learn.microsoft.com/en-us/graph/api/team-list?view=graph-rest-1.0&tabs=http +func AddTeamFlag(cmd *cobra.Command) { + cmd.Flags().StringSliceVar( + &TeamFV, + TeamFN, nil, + "Backup data by team; accepts '"+Wildcard+"' to select all teams.") +} diff --git a/src/cli/restore/groups.go b/src/cli/restore/groups.go new file mode 100644 index 000000000..a98c9d088 --- /dev/null +++ b/src/cli/restore/groups.go @@ -0,0 +1,81 @@ +package restore + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by restore.go to map subcommands to provider-specific handling. +func addGroupsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case restoreCommand: + c, fs = utils.AddCommand(cmd, groupsRestoreCmd(), utils.HideCommand()) + + c.Use = c.Use + " " + groupsServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddRestorePermissionsFlag(c) + flags.AddRestoreConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + } + + return c +} + +// TODO: correct examples +const ( + groupsServiceCommand = "groups" + groupsServiceCommandUseSuffix = "--backup " + + groupsServiceCommandRestoreExamples = `# Restore file with ID 98765abcdef in Bob's last backup (1234abcd...) +corso restore groups --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Restore the file with ID 98765abcdef along with its associated permissions +corso restore groups --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef --restore-permissions + +# Restore files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" +corso restore groups --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Restore all files and folders in folder "Documents/Finance Reports" that were created before 2020 +corso restore groups --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso restore groups [...]` +func groupsRestoreCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "Restore M365 Groups service data", + RunE: restoreGroupsCmd, + Args: cobra.NoArgs, + Example: groupsServiceCommandRestoreExamples, + } +} + +// processes an groups service restore. +func restoreGroupsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} diff --git a/src/cli/restore/groups_test.go b/src/cli/restore/groups_test.go new file mode 100644 index 000000000..4ea7a7d19 --- /dev/null +++ b/src/cli/restore/groups_test.go @@ -0,0 +1,108 @@ +package restore + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type GroupsUnitSuite struct { + tester.Suite +} + +func TestGroupsUnitSuite(t *testing.T) { + suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *GroupsUnitSuite) TestAddGroupsCommands() { + expectUse := groupsServiceCommand + " " + groupsServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"restore groups", restoreCommand, expectUse, groupsRestoreCmd().Short, restoreGroupsCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addGroupsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "groups", + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + + "--" + flags.CollisionsFN, testdata.Collisions, + "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.AzureClientIDFN, testdata.AzureClientID, + "--" + flags.AzureClientTenantFN, testdata.AzureTenantID, + "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + // assert.NoError(t, err, clues.ToCore(err)) + assert.ErrorIs(t, err, utils.ErrNotYetImplemented, clues.ToCore(err)) + + opts := utils.MakeGroupsOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) + assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.AzureClientID, flags.AzureClientIDFV) + assert.Equal(t, testdata.AzureTenantID, flags.AzureClientTenantFV) + assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + assert.True(t, flags.RestorePermissionsFV) + }) + } +} diff --git a/src/cli/restore/teams.go b/src/cli/restore/teams.go new file mode 100644 index 000000000..59623024a --- /dev/null +++ b/src/cli/restore/teams.go @@ -0,0 +1,81 @@ +package restore + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by restore.go to map subcommands to provider-specific handling. +func addTeamsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case restoreCommand: + c, fs = utils.AddCommand(cmd, teamsRestoreCmd(), utils.HideCommand()) + + c.Use = c.Use + " " + teamsServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddRestorePermissionsFlag(c) + flags.AddRestoreConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + flags.AddAzureCredsFlags(c) + } + + return c +} + +// TODO: correct examples +const ( + teamsServiceCommand = "teams" + teamsServiceCommandUseSuffix = "--backup " + + teamsServiceCommandRestoreExamples = `# Restore file with ID 98765abcdef in Bob's last backup (1234abcd...) +corso restore teams --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Restore the file with ID 98765abcdef along with its associated permissions +corso restore teams --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef --restore-permissions + +# Restore files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" +corso restore teams --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Restore all files and folders in folder "Documents/Finance Reports" that were created before 2020 +corso restore teams --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso restore teams [...]` +func teamsRestoreCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "Restore M365 Teams service data", + RunE: restoreTeamsCmd, + Args: cobra.NoArgs, + Example: teamsServiceCommandRestoreExamples, + } +} + +// processes an teams service restore. +func restoreTeamsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} diff --git a/src/cli/restore/teams_test.go b/src/cli/restore/teams_test.go new file mode 100644 index 000000000..ac502e950 --- /dev/null +++ b/src/cli/restore/teams_test.go @@ -0,0 +1,108 @@ +package restore + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type TeamsUnitSuite struct { + tester.Suite +} + +func TestTeamsUnitSuite(t *testing.T) { + suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *TeamsUnitSuite) TestAddTeamsCommands() { + expectUse := teamsServiceCommand + " " + teamsServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"restore teams", restoreCommand, expectUse, teamsRestoreCmd().Short, restoreTeamsCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addTeamsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "teams", + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + + "--" + flags.CollisionsFN, testdata.Collisions, + "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.AzureClientIDFN, testdata.AzureClientID, + "--" + flags.AzureClientTenantFN, testdata.AzureTenantID, + "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + // assert.NoError(t, err, clues.ToCore(err)) + assert.ErrorIs(t, err, utils.ErrNotYetImplemented, clues.ToCore(err)) + + opts := utils.MakeTeamsOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) + assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.AzureClientID, flags.AzureClientIDFV) + assert.Equal(t, testdata.AzureTenantID, flags.AzureClientTenantFV) + assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + assert.True(t, flags.RestorePermissionsFV) + }) + } +} diff --git a/src/cli/utils/groups.go b/src/cli/utils/groups.go new file mode 100644 index 000000000..9b0827d46 --- /dev/null +++ b/src/cli/utils/groups.go @@ -0,0 +1,30 @@ +package utils + +import ( + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" +) + +type GroupsOpts struct { + Groups []string + + RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts + + Populated flags.PopulatedFlags +} + +func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts { + return GroupsOpts{ + Groups: flags.UserFV, + + RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} diff --git a/src/cli/utils/teams.go b/src/cli/utils/teams.go new file mode 100644 index 000000000..365e7971e --- /dev/null +++ b/src/cli/utils/teams.go @@ -0,0 +1,30 @@ +package utils + +import ( + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" +) + +type TeamsOpts struct { + Teams []string + + RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts + + Populated flags.PopulatedFlags +} + +func MakeTeamsOpts(cmd *cobra.Command) TeamsOpts { + return TeamsOpts{ + Teams: flags.UserFV, + + RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index a542d55f3..5a639474a 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -19,6 +19,8 @@ import ( "github.com/alcionai/corso/src/pkg/storage" ) +var ErrNotYetImplemented = clues.New("not yet implemented") + func GetAccountAndConnect( ctx context.Context, pst path.ServiceType, diff --git a/src/pkg/path/service_type.go b/src/pkg/path/service_type.go index 0028bca4b..343117857 100644 --- a/src/pkg/path/service_type.go +++ b/src/pkg/path/service_type.go @@ -31,6 +31,8 @@ const ( SharePointMetadataService // sharepointMetadata GroupsService // groups GroupsMetadataService // groupsMetadata + TeamsService // teams + TeamsMetadataService // teamsMetadata ) func toServiceType(service string) ServiceType { From 9abd9d4f96312a2dc5314d60a740ee6670ebb282 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 18 Aug 2023 16:45:21 -0600 Subject: [PATCH 04/16] remove all uses of iota (#4046) I've needed to catch gotchas that arise from contributors adding a value in the middle of an iota list, not to mention have dealt with prior bugs that happened the same way, now too many times to feel safe about its usage. This PR removes the use of iota from all const declarations. The intent is to not allow the use of iota within the codebase. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3993 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/data/implementations.go | 8 ++--- .../m365/collection/drive/collections.go | 6 ++-- .../collection/drive/metadata/permissions.go | 4 +-- src/internal/m365/collection/site/backup.go | 6 ++-- .../m365/collection/site/backup_test.go | 4 +++ .../m365/collection/site/collection.go | 26 +++++++++------- .../m365/collection/site/collection_test.go | 11 ++++--- .../collection/site/datacategory_string.go | 27 ----------------- .../m365/service/sharepoint/backup.go | 2 ++ src/internal/m365/support/status.go | 8 ++--- src/internal/model/model.go | 14 ++++----- src/internal/operations/operation.go | 10 +++---- src/pkg/account/account.go | 4 +-- src/pkg/backup/details/iteminfo.go | 15 +++++----- src/pkg/control/repository/repo.go | 12 +++----- src/pkg/path/category_type.go | 18 +++++------ src/pkg/path/service_type.go | 30 ++++++++++++------- src/pkg/path/servicetype_string.go | 6 ++-- src/pkg/selectors/selectors.go | 10 +++---- src/pkg/storage/storage.go | 4 +-- 20 files changed, 110 insertions(+), 115 deletions(-) delete mode 100644 src/internal/m365/collection/site/datacategory_string.go diff --git a/src/internal/data/implementations.go b/src/internal/data/implementations.go index 15b7dffb3..d75bd93b6 100644 --- a/src/internal/data/implementations.go +++ b/src/internal/data/implementations.go @@ -13,10 +13,10 @@ var ErrNotFound = clues.New("not found") type CollectionState int const ( - NewState = CollectionState(iota) - NotMovedState - MovedState - DeletedState + NewState CollectionState = 0 + NotMovedState CollectionState = 1 + MovedState CollectionState = 2 + DeletedState CollectionState = 3 ) type FetchRestoreCollection struct { diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index a2161f779..6964774b8 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -31,13 +31,13 @@ type collectionScope int const ( // CollectionScopeUnknown is used when we don't know and don't need // to know the kind, like in the case of deletes - CollectionScopeUnknown collectionScope = iota + CollectionScopeUnknown collectionScope = 0 // CollectionScopeFolder is used for regular folder collections - CollectionScopeFolder + CollectionScopeFolder collectionScope = 1 // CollectionScopePackage is used to represent OneNote items - CollectionScopePackage + CollectionScopePackage collectionScope = 2 ) const restrictedDirectory = "Site Pages" diff --git a/src/internal/m365/collection/drive/metadata/permissions.go b/src/internal/m365/collection/drive/metadata/permissions.go index ec0cc22f0..53f549110 100644 --- a/src/internal/m365/collection/drive/metadata/permissions.go +++ b/src/internal/m365/collection/drive/metadata/permissions.go @@ -14,8 +14,8 @@ import ( type SharingMode int const ( - SharingModeCustom = SharingMode(iota) - SharingModeInherited + SharingModeCustom SharingMode = 0 + SharingModeInherited SharingMode = 1 ) type GV2Type string diff --git a/src/internal/m365/collection/site/backup.go b/src/internal/m365/collection/site/backup.go index 14f1333be..8357d9512 100644 --- a/src/internal/m365/collection/site/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -59,6 +59,7 @@ func CollectPages( bpc inject.BackupProducerConfig, creds account.M365Config, ac api.Client, + scope selectors.SharePointScope, su support.StatusUpdater, errs *fault.Bus, ) ([]data.BackupCollection, error) { @@ -105,7 +106,7 @@ func CollectPages( collection := NewCollection( dir, ac, - Pages, + scope, su, bpc.Options) collection.SetBetaService(betaService) @@ -122,6 +123,7 @@ func CollectLists( bpc inject.BackupProducerConfig, ac api.Client, tenantID string, + scope selectors.SharePointScope, su support.StatusUpdater, errs *fault.Bus, ) ([]data.BackupCollection, error) { @@ -156,7 +158,7 @@ func CollectLists( collection := NewCollection( dir, ac, - List, + scope, su, bpc.Options) collection.AddJob(tuple.ID) diff --git a/src/internal/m365/collection/site/backup_test.go b/src/internal/m365/collection/site/backup_test.go index de0d91c50..46dff1a97 100644 --- a/src/internal/m365/collection/site/backup_test.go +++ b/src/internal/m365/collection/site/backup_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -61,11 +62,14 @@ func (suite *SharePointPagesSuite) TestCollectPages() { ProtectedResource: mock.NewProvider(siteID, siteID), } + sel := selectors.NewSharePointBackup([]string{siteID}) + col, err := CollectPages( ctx, bpc, creds, ac, + sel.Lists(selectors.Any())[0], (&MockGraphService{}).UpdateStatus, fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index a293e40a0..a6196a4ed 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -21,19 +21,23 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" ) type DataCategory int +// channel sizes +const ( + collectionChannelBufferSize = 50 + fetchChannelSize = 5 +) + //go:generate stringer -type=DataCategory const ( - collectionChannelBufferSize = 50 - fetchChannelSize = 5 - Unknown DataCategory = iota - List - Drive - Pages + Unknown DataCategory = 0 + List DataCategory = 1 + Pages DataCategory = 2 ) var ( @@ -53,7 +57,7 @@ type Collection struct { // jobs contain the SharePoint.Site.ListIDs for the associated list(s). jobs []string // M365 IDs of the items of this collection - category DataCategory + category path.CategoryType client api.Sites ctrl control.Options betaService *betaAPI.BetaService @@ -64,7 +68,7 @@ type Collection struct { func NewCollection( folderPath path.Path, ac api.Client, - category DataCategory, + scope selectors.SharePointScope, statusUpdater support.StatusUpdater, ctrlOpts control.Options, ) *Collection { @@ -74,7 +78,7 @@ func NewCollection( data: make(chan data.Item, collectionChannelBufferSize), client: ac.Sites(), statusUpdater: statusUpdater, - category: category, + category: scope.Category().PathType(), ctrl: ctrlOpts, } @@ -198,9 +202,9 @@ func (sc *Collection) runPopulate( // Switch retrieval function based on category switch sc.category { - case List: + case path.ListsCategory: metrics, err = sc.retrieveLists(ctx, writer, colProgress, errs) - case Pages: + case path.PagesCategory: metrics, err = sc.retrievePages(ctx, sc.client, writer, colProgress, errs) } diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index f3f19c7e4..390d5cd14 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -23,6 +23,7 @@ import ( "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -82,16 +83,18 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { dirRoot = "directory" ) + sel := selectors.NewSharePointBackup([]string{"site"}) + tables := []struct { name, itemName string - category DataCategory + scope selectors.SharePointScope getDir func(t *testing.T) path.Path getItem func(t *testing.T, itemName string) *Item }{ { name: "List", itemName: "MockListing", - category: List, + scope: sel.Lists(selectors.Any())[0], getDir: func(t *testing.T) path.Path { dir, err := path.Build( tenant, @@ -127,7 +130,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { { name: "Pages", itemName: "MockPages", - category: Pages, + scope: sel.Pages(selectors.Any())[0], getDir: func(t *testing.T) path.Path { dir, err := path.Build( tenant, @@ -166,7 +169,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { col := NewCollection( test.getDir(t), suite.ac, - test.category, + test.scope, nil, control.DefaultOptions()) col.data <- test.getItem(t, test.itemName) diff --git a/src/internal/m365/collection/site/datacategory_string.go b/src/internal/m365/collection/site/datacategory_string.go deleted file mode 100644 index eac0006cc..000000000 --- a/src/internal/m365/collection/site/datacategory_string.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated by "stringer -type=DataCategory"; DO NOT EDIT. - -package site - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[Unknown-2] - _ = x[List-3] - _ = x[Drive-4] - _ = x[Pages-5] -} - -const _DataCategory_name = "UnknownListDrivePages" - -var _DataCategory_index = [...]uint8{0, 7, 11, 16, 21} - -func (i DataCategory) String() string { - i -= 2 - if i < 0 || i >= DataCategory(len(_DataCategory_index)-1) { - return "DataCategory(" + strconv.FormatInt(int64(i+2), 10) + ")" - } - return _DataCategory_name[_DataCategory_index[i]:_DataCategory_index[i+1]] -} diff --git a/src/internal/m365/service/sharepoint/backup.go b/src/internal/m365/service/sharepoint/backup.go index 479d4ac24..c4604e609 100644 --- a/src/internal/m365/service/sharepoint/backup.go +++ b/src/internal/m365/service/sharepoint/backup.go @@ -63,6 +63,7 @@ func ProduceBackupCollections( bpc, ac, creds.AzureTenantID, + scope, su, errs) if err != nil { @@ -95,6 +96,7 @@ func ProduceBackupCollections( bpc, creds, ac, + scope, su, errs) if err != nil { diff --git a/src/internal/m365/support/status.go b/src/internal/m365/support/status.go index b1a7d2449..5e85857eb 100644 --- a/src/internal/m365/support/status.go +++ b/src/internal/m365/support/status.go @@ -37,10 +37,10 @@ type Operation int //go:generate stringer -type=Operation const ( - OpUnknown Operation = iota - Backup - Restore - Export + OpUnknown Operation = 0 + Backup Operation = 1 + Restore Operation = 2 + Export Operation = 3 ) // Constructor for ConnectorOperationStatus. If the counts do not agree, an error is returned. diff --git a/src/internal/model/model.go b/src/internal/model/model.go index dcf0dce51..a3f25c820 100644 --- a/src/internal/model/model.go +++ b/src/internal/model/model.go @@ -22,12 +22,12 @@ func (id StableID) String() string { // //go:generate go run golang.org/x/tools/cmd/stringer -type=Schema const ( - UnknownSchema = Schema(iota) - BackupOpSchema - RestoreOpSchema - BackupSchema - BackupDetailsSchema - RepositorySchema + UnknownSchema Schema = 0 + BackupOpSchema Schema = 1 + RestoreOpSchema Schema = 2 + BackupSchema Schema = 3 + BackupDetailsSchema Schema = 4 + RepositorySchema Schema = 5 ) // common tags for filtering @@ -38,7 +38,7 @@ const ( MergeBackup = "merge-backup" ) -// Valid returns true if the ModelType value fits within the iota range. +// Valid returns true if the ModelType value fits within the const range. func (mt Schema) Valid() bool { return mt > 0 && mt < RepositorySchema+1 } diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index 35bf9fb19..c400e52cd 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -33,11 +33,11 @@ type OpStatus int //go:generate stringer -type=OpStatus -linecomment const ( - Unknown OpStatus = iota // Status Unknown - InProgress // In Progress - Completed // Completed - Failed // Failed - NoData // No Data + Unknown OpStatus = 0 // Status Unknown + InProgress OpStatus = 1 // In Progress + Completed OpStatus = 2 // Completed + Failed OpStatus = 3 // Failed + NoData OpStatus = 4 // No Data ) // -------------------------------------------------------------------------------- diff --git a/src/pkg/account/account.go b/src/pkg/account/account.go index 12b8d679c..4c1591818 100644 --- a/src/pkg/account/account.go +++ b/src/pkg/account/account.go @@ -10,8 +10,8 @@ type accountProvider int //go:generate stringer -type=accountProvider -linecomment const ( - ProviderUnknown accountProvider = iota // Unknown Provider - ProviderM365 // M365 + ProviderUnknown accountProvider = 0 // Unknown Provider + ProviderM365 accountProvider = 1 // M365 ) // storage parsing errors diff --git a/src/pkg/backup/details/iteminfo.go b/src/pkg/backup/details/iteminfo.go index 9912fb6d2..fbd6a92cd 100644 --- a/src/pkg/backup/details/iteminfo.go +++ b/src/pkg/backup/details/iteminfo.go @@ -20,16 +20,17 @@ type ItemType int // Additionally, any itemType directly assigned a number should not be altered. // This applies to OneDriveItem and FolderItem const ( - UnknownType ItemType = iota // 0, global unknown value + UnknownType ItemType = 0 // Exchange (00x) - ExchangeContact - ExchangeEvent - ExchangeMail + ExchangeContact ItemType = 1 + ExchangeEvent ItemType = 2 + ExchangeMail ItemType = 3 + // SharePoint (10x) - SharePointLibrary ItemType = iota + 97 // 100 - SharePointList // 101... - SharePointPage + SharePointLibrary ItemType = 101 + SharePointList ItemType = 102 + SharePointPage ItemType = 103 // OneDrive (20x) OneDriveItem ItemType = 205 diff --git a/src/pkg/control/repository/repo.go b/src/pkg/control/repository/repo.go index 0d80a1fda..6d1869f91 100644 --- a/src/pkg/control/repository/repo.go +++ b/src/pkg/control/repository/repo.go @@ -25,12 +25,10 @@ type Maintenance struct { type MaintenanceType int -// Can't be reordered as we rely on iota for numbering. -// //go:generate stringer -type=MaintenanceType -linecomment const ( - CompleteMaintenance MaintenanceType = iota // complete - MetadataMaintenance // metadata + CompleteMaintenance MaintenanceType = 0 // complete + MetadataMaintenance MaintenanceType = 1 // metadata ) var StringToMaintenanceType = map[string]MaintenanceType{ @@ -40,16 +38,14 @@ var StringToMaintenanceType = map[string]MaintenanceType{ type MaintenanceSafety int -// Can't be reordered as we rely on iota for numbering. -// //go:generate stringer -type=MaintenanceSafety -linecomment const ( - FullMaintenanceSafety MaintenanceSafety = iota + FullMaintenanceSafety MaintenanceSafety = 0 //nolint:lll // Use only if there's no other kopia instances accessing the repo and the // storage backend is strongly consistent. // https://github.com/kopia/kopia/blob/f9de453efc198b6e993af8922f953a7e5322dc5f/repo/maintenance/maintenance_safety.go#L42 - NoMaintenanceSafety + NoMaintenanceSafety MaintenanceSafety = 1 ) type RetentionMode int diff --git a/src/pkg/path/category_type.go b/src/pkg/path/category_type.go index 4a992176f..5f8009e5d 100644 --- a/src/pkg/path/category_type.go +++ b/src/pkg/path/category_type.go @@ -17,15 +17,15 @@ type CategoryType int //go:generate stringer -type=CategoryType -linecomment const ( - UnknownCategory CategoryType = iota - EmailCategory // email - ContactsCategory // contacts - EventsCategory // events - FilesCategory // files - ListsCategory // lists - LibrariesCategory // libraries - PagesCategory // pages - DetailsCategory // details + UnknownCategory CategoryType = 0 + EmailCategory CategoryType = 1 // email + ContactsCategory CategoryType = 2 // contacts + EventsCategory CategoryType = 3 // events + FilesCategory CategoryType = 4 // files + ListsCategory CategoryType = 5 // lists + LibrariesCategory CategoryType = 6 // libraries + PagesCategory CategoryType = 7 // pages + DetailsCategory CategoryType = 8 // details ) func ToCategoryType(category string) CategoryType { diff --git a/src/pkg/path/service_type.go b/src/pkg/path/service_type.go index 343117857..a4a99ec6c 100644 --- a/src/pkg/path/service_type.go +++ b/src/pkg/path/service_type.go @@ -22,17 +22,17 @@ type ServiceType int //go:generate stringer -type=ServiceType -linecomment const ( - UnknownService ServiceType = iota - ExchangeService // exchange - OneDriveService // onedrive - SharePointService // sharepoint - ExchangeMetadataService // exchangeMetadata - OneDriveMetadataService // onedriveMetadata - SharePointMetadataService // sharepointMetadata - GroupsService // groups - GroupsMetadataService // groupsMetadata - TeamsService // teams - TeamsMetadataService // teamsMetadata + UnknownService ServiceType = 0 + ExchangeService ServiceType = 1 // exchange + OneDriveService ServiceType = 2 // onedrive + SharePointService ServiceType = 3 // sharepoint + ExchangeMetadataService ServiceType = 4 // exchangeMetadata + OneDriveMetadataService ServiceType = 5 // onedriveMetadata + SharePointMetadataService ServiceType = 6 // sharepointMetadata + GroupsService ServiceType = 7 // groups + GroupsMetadataService ServiceType = 8 // groupsMetadata + TeamsService ServiceType = 9 // teams + TeamsMetadataService ServiceType = 10 // teamsMetadata ) func toServiceType(service string) ServiceType { @@ -45,12 +45,20 @@ func toServiceType(service string) ServiceType { return OneDriveService case strings.ToLower(SharePointService.String()): return SharePointService + case strings.ToLower(GroupsService.String()): + return GroupsService + case strings.ToLower(TeamsService.String()): + return TeamsService case strings.ToLower(ExchangeMetadataService.String()): return ExchangeMetadataService case strings.ToLower(OneDriveMetadataService.String()): return OneDriveMetadataService case strings.ToLower(SharePointMetadataService.String()): return SharePointMetadataService + case strings.ToLower(GroupsMetadataService.String()): + return GroupsMetadataService + case strings.ToLower(TeamsMetadataService.String()): + return TeamsMetadataService default: return UnknownService } diff --git a/src/pkg/path/servicetype_string.go b/src/pkg/path/servicetype_string.go index 6fa499364..4b9ab16ec 100644 --- a/src/pkg/path/servicetype_string.go +++ b/src/pkg/path/servicetype_string.go @@ -17,11 +17,13 @@ func _() { _ = x[SharePointMetadataService-6] _ = x[GroupsService-7] _ = x[GroupsMetadataService-8] + _ = x[TeamsService-9] + _ = x[TeamsMetadataService-10] } -const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadatagroupsgroupsMetadata" +const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadatagroupsgroupsMetadatateamsteamsMetadata" -var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90, 96, 110} +var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90, 96, 110, 115, 128} func (i ServiceType) String() string { if i < 0 || i >= ServiceType(len(_ServiceType_index)-1) { diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index ac85f75c3..3a18c2bd0 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -20,11 +20,11 @@ type service int //go:generate stringer -type=service -linecomment const ( - ServiceUnknown service = iota // Unknown Service - ServiceExchange // Exchange - ServiceOneDrive // OneDrive - ServiceSharePoint // SharePoint - ServiceGroups // Groups + ServiceUnknown service = 0 // Unknown Service + ServiceExchange service = 1 // Exchange + ServiceOneDrive service = 2 // OneDrive + ServiceSharePoint service = 3 // SharePoint + ServiceGroups service = 4 // Groups ) var serviceToPathType = map[service]path.ServiceType{ diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index 673503587..e197f4081 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -12,8 +12,8 @@ type storageProvider int //go:generate stringer -type=storageProvider -linecomment const ( - ProviderUnknown storageProvider = iota // Unknown Provider - ProviderS3 // S3 + ProviderUnknown storageProvider = 0 // Unknown Provider + ProviderS3 storageProvider = 1 // S3 ) // storage parsing errors From 2ba349797f50bf1afdb24843e4395d31ce21a186 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 05:54:57 +0000 Subject: [PATCH 05/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20sass=20from?= =?UTF-8?q?=201.65.1=20to=201.66.1=20in=20/website=20(#4072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sass](https://github.com/sass/dart-sass) from 1.65.1 to 1.66.1.
Release notes

Sourced from sass's releases.

Dart Sass 1.66.1

To install Sass 1.66.1, download one of the packages below and add it to your PATH, or see the Sass website for full installation instructions.

Changes

JS API

  • Fix a bug where Sass compilation could crash in strict mode if passed a callback that threw a string, boolean, number, symbol, or bignum.

See the full changelog for changes in earlier releases.

Dart Sass 1.66.0

To install Sass 1.66.0, download one of the packages below and add it to your PATH, or see the Sass website for full installation instructions.

Changes

  • Breaking change: Drop support for the additional CSS calculations defined in CSS Values and Units 4. Custom Sass functions whose names overlapped with these new CSS functions were being parsed as CSS calculations instead, causing an unintentional breaking change outside our normal [compatibility policy] for CSS compatibility changes.

    Support will be added again in a future version, but only after Sass has emitted a deprecation warning for all functions that will break for at least three months prior to the breakage.

See the full changelog for changes in earlier releases.

Changelog

Sourced from sass's changelog.

1.66.1

JS API

  • Fix a bug where Sass compilation could crash in strict mode if passed a callback that threw a string, boolean, number, symbol, or bignum.

1.66.0

  • Breaking change: Drop support for the additional CSS calculations defined in CSS Values and Units 4. Custom Sass functions whose names overlapped with these new CSS functions were being parsed as CSS calculations instead, causing an unintentional breaking change outside our normal [compatibility policy] for CSS compatibility changes.

    Support will be added again in a future version, but only after Sass has emitted a deprecation warning for all functions that will break for at least three months prior to the breakage.

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sass&package-manager=npm_and_yarn&previous-version=1.65.1&new-version=1.66.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 14 +++++++------- website/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index ef8a2cc4f..581e44381 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -24,7 +24,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.65.1", + "sass": "^1.66.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -12639,9 +12639,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.65.1", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.65.1.tgz", - "integrity": "sha512-9DINwtHmA41SEd36eVPQ9BJKpn7eKDQmUHmpI0y5Zv2Rcorrh0zS+cFrt050hdNbmmCNKTW3hV5mWfuegNRsEA==", + "version": "1.66.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.66.1.tgz", + "integrity": "sha512-50c+zTsZOJVgFfTgwwEzkjA3/QACgdNsKueWPyAR0mRINIvLAStVQBbPg14iuqEQ74NPDbXzJARJ/O4SI1zftA==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -23932,9 +23932,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass": { - "version": "1.65.1", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.65.1.tgz", - "integrity": "sha512-9DINwtHmA41SEd36eVPQ9BJKpn7eKDQmUHmpI0y5Zv2Rcorrh0zS+cFrt050hdNbmmCNKTW3hV5mWfuegNRsEA==", + "version": "1.66.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.66.1.tgz", + "integrity": "sha512-50c+zTsZOJVgFfTgwwEzkjA3/QACgdNsKueWPyAR0mRINIvLAStVQBbPg14iuqEQ74NPDbXzJARJ/O4SI1zftA==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/website/package.json b/website/package.json index 5cbecd8c2..7528e4759 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.65.1", + "sass": "^1.66.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" From b37ee8aced0066a4cc1bd5d71c9068250f64f2e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 06:51:17 +0000 Subject: [PATCH 06/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.326=20to=201.44.327=20in=20/src=20(#?= =?UTF-8?q?4073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.326 to 1.44.327.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.327 (2023-08-18)

Service Client Updates

  • service/codecommit: Updates service API, documentation, and paginators
    • Add new ListFileCommitHistory operation to retrieve commits which introduced changes to a specific file.
  • service/securityhub: Updates service API and documentation

SDK Bugs

  • aws/credentials/ssocreds: Modify sso token provider logic to handle possible nil val returned by CreateToken.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.326&new-version=1.44.327)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 7dc1f418a..af8b05608 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.326 + github.com/aws/aws-sdk-go v1.44.327 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index fd1a66ad1..4122060fb 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.326 h1:/6xD/9mKZ2RMTDfbhh9qCxw+CaTbJRvfHJ/NHPFbI38= -github.com/aws/aws-sdk-go v1.44.326/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= +github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 13e0d82735464ffb3e6fd9b4f45e74f87d2ac7e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:22:17 +0000 Subject: [PATCH 07/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/mi?= =?UTF-8?q?crosoftgraph/msgraph-sdk-go=20from=201.14.0=20to=201.15.0=20in?= =?UTF-8?q?=20/src=20(#4075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/microsoftgraph/msgraph-sdk-go](https://github.com/microsoftgraph/msgraph-sdk-go) from 1.14.0 to 1.15.0.
Changelog

Sourced from github.com/microsoftgraph/msgraph-sdk-go's changelog.

[1.15.0]- 2023-08-21

Changed

  • Weekly generation.
Commits
  • 4b0d8c6 Generated models and request builders (#554)
  • 344b8dd Merge pull request #553 from microsoftgraph/dependabot/go_modules/github.com/...
  • e3bb680 Bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.7.0 to 1.7.1
  • 82bbc80 Merge pull request #549 from microsoftgraph/dependabot/go_modules/github.com/...
  • 1f12cf9 Bump github.com/microsoft/kiota-abstractions-go from 1.1.0 to 1.2.0
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/microsoftgraph/msgraph-sdk-go&package-manager=go_modules&previous-version=1.14.0&new-version=1.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 3 ++- src/go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index af8b05608..00231cdfc 100644 --- a/src/go.mod +++ b/src/go.mod @@ -19,7 +19,7 @@ require ( github.com/microsoft/kiota-http-go v1.1.0 github.com/microsoft/kiota-serialization-form-go v1.0.0 github.com/microsoft/kiota-serialization-json-go v1.0.4 - github.com/microsoftgraph/msgraph-sdk-go v1.14.0 + github.com/microsoftgraph/msgraph-sdk-go v1.15.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 @@ -49,6 +49,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/microsoft/kiota-serialization-multipart-go v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.9 // indirect github.com/spf13/afero v1.9.5 // indirect diff --git a/src/go.sum b/src/go.sum index 4122060fb..f5263b371 100644 --- a/src/go.sum +++ b/src/go.sum @@ -281,10 +281,12 @@ github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjb github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= github.com/microsoft/kiota-serialization-json-go v1.0.4/go.mod h1:rM4+FsAY+9AEpBsBzkFFis+b/LZLlNKKewuLwK9Q6Mg= +github.com/microsoft/kiota-serialization-multipart-go v1.0.0 h1:3O5sb5Zj+moLBiJympbXNaeV07K0d46IfuEd5v9+pBs= +github.com/microsoft/kiota-serialization-multipart-go v1.0.0/go.mod h1:yauLeBTpANk4L03XD985akNysG24SnRJGaveZf+p4so= github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA= github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M= -github.com/microsoftgraph/msgraph-sdk-go v1.14.0 h1:YdhMvzu8bXcfIQGRur6NkXnv4cPOsMBJ44XjfWLOt9Y= -github.com/microsoftgraph/msgraph-sdk-go v1.14.0/go.mod h1:ccLv84FJFtwdSzYWM/HlTes5FLzkzzBsYh9kg93/WS8= +github.com/microsoftgraph/msgraph-sdk-go v1.15.0 h1:cdz6Bs0T0Hl/NTdUAZq8TRJwidTmX741X2SnVIsn5l4= +github.com/microsoftgraph/msgraph-sdk-go v1.15.0/go.mod h1:YfKdWdUwQWuS6E+Qg6+SZnHxJ/kvG2nYQutwzGa5NZs= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8wCKPgpODs/D1cUZ3fkAY= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= From 734b1021c90940aab2e14553d03924380cad4c24 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 21 Aug 2023 09:58:48 -0700 Subject: [PATCH 08/16] Fix log output and don't fail fast (#4076) #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/nightly_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index f6338a4c1..a676a5bac 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -92,7 +92,7 @@ jobs: CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} - CORSO_LOG_FILE: ${{ github.workspace }}/testlog/run-nightly.log + CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-nightly.log LOG_GRAPH_REQUESTS: true S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} run: | @@ -101,7 +101,6 @@ jobs: -tags testing \ -json \ -v \ - -failfast \ -p 1 \ -timeout 1h \ ./... 2>&1 | tee ./testlog/gotest-nightly.log | gotestfmt -hide successful-tests From 1468c0881aa1d963e540b67705ab5ac420315493 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 18:14:57 +0000 Subject: [PATCH 09/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/pu?= =?UTF-8?q?zpuzpuz/xsync/v2=20from=202.4.1=20to=202.5.0=20in=20/src=20(#40?= =?UTF-8?q?77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/puzpuzpuz/xsync/v2](https://github.com/puzpuzpuz/xsync) from 2.4.1 to 2.5.0.
Release notes

Sourced from github.com/puzpuzpuz/xsync/v2's releases.

v2.5.0

  • Add concurrent queue with generics support (MPMCQueueOf) (#104)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/puzpuzpuz/xsync/v2&package-manager=go_modules&previous-version=2.4.1&new-version=2.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 00231cdfc..e34158eb9 100644 --- a/src/go.mod +++ b/src/go.mod @@ -22,7 +22,7 @@ require ( github.com/microsoftgraph/msgraph-sdk-go v1.15.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 - github.com/puzpuzpuz/xsync/v2 v2.4.1 + github.com/puzpuzpuz/xsync/v2 v2.5.0 github.com/rudderlabs/analytics-go v3.3.3+incompatible github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 github.com/spf13/cobra v1.7.0 diff --git a/src/go.sum b/src/go.sum index f5263b371..aadbdc6f3 100644 --- a/src/go.sum +++ b/src/go.sum @@ -344,8 +344,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/puzpuzpuz/xsync/v2 v2.4.1 h1:aGdE1C/HaR/QC6YAFdtZXi60Df8/qBIrs8PKrzkItcM= -github.com/puzpuzpuz/xsync/v2 v2.4.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= +github.com/puzpuzpuz/xsync/v2 v2.5.0 h1:2k4qrO/orvmEXZ3hmtHqIy9XaQtPTwzMZk1+iErpE8c= +github.com/puzpuzpuz/xsync/v2 v2.5.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From 0a875907699b82428aeb4f281f851c1282370fc6 Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 21 Aug 2023 12:49:04 -0600 Subject: [PATCH 10/16] Teams groups export cli (#4069) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3989 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- src/cli/backup/groups.go | 8 +-- src/cli/backup/teams.go | 8 +-- src/cli/export/export.go | 2 + src/cli/export/groups.go | 84 +++++++++++++++++++++++++++++++ src/cli/export/groups_test.go | 94 +++++++++++++++++++++++++++++++++++ src/cli/export/onedrive.go | 4 +- src/cli/export/sharepoint.go | 4 +- src/cli/export/teams.go | 84 +++++++++++++++++++++++++++++++ src/cli/export/teams_test.go | 94 +++++++++++++++++++++++++++++++++++ src/cli/restore/groups.go | 2 +- src/cli/restore/teams.go | 2 +- 11 files changed, 372 insertions(+), 14 deletions(-) create mode 100644 src/cli/export/groups.go create mode 100644 src/cli/export/groups_test.go create mode 100644 src/cli/export/teams.go create mode 100644 src/cli/export/teams_test.go diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go index 3f1f83eb7..1dc490ae7 100644 --- a/src/cli/backup/groups.go +++ b/src/cli/backup/groups.go @@ -53,7 +53,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case createCommand: - c, fs = utils.AddCommand(cmd, groupsCreateCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + groupsServiceCommandCreateUseSuffix @@ -69,7 +69,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command { flags.AddFailFastFlag(c) case listCommand: - c, fs = utils.AddCommand(cmd, groupsListCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false flags.AddBackupIDFlag(c, false) @@ -81,7 +81,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command { addRecoveredErrorsFN(c) case detailsCommand: - c, fs = utils.AddCommand(cmd, groupsDetailsCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + groupsServiceCommandDetailsUseSuffix @@ -97,7 +97,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command { flags.AddAzureCredsFlags(c) case deleteCommand: - c, fs = utils.AddCommand(cmd, groupsDeleteCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + groupsServiceCommandDeleteUseSuffix diff --git a/src/cli/backup/teams.go b/src/cli/backup/teams.go index fcac3394d..97e314cfd 100644 --- a/src/cli/backup/teams.go +++ b/src/cli/backup/teams.go @@ -53,7 +53,7 @@ func addTeamsCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case createCommand: - c, fs = utils.AddCommand(cmd, teamsCreateCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, teamsCreateCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + teamsServiceCommandCreateUseSuffix @@ -69,7 +69,7 @@ func addTeamsCommands(cmd *cobra.Command) *cobra.Command { flags.AddFailFastFlag(c) case listCommand: - c, fs = utils.AddCommand(cmd, teamsListCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, teamsListCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false flags.AddBackupIDFlag(c, false) @@ -81,7 +81,7 @@ func addTeamsCommands(cmd *cobra.Command) *cobra.Command { addRecoveredErrorsFN(c) case detailsCommand: - c, fs = utils.AddCommand(cmd, teamsDetailsCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, teamsDetailsCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + teamsServiceCommandDetailsUseSuffix @@ -97,7 +97,7 @@ func addTeamsCommands(cmd *cobra.Command) *cobra.Command { flags.AddAzureCredsFlags(c) case deleteCommand: - c, fs = utils.AddCommand(cmd, teamsDeleteCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, teamsDeleteCmd(), utils.MarkPreReleaseCommand()) fs.SortFlags = false c.Use = c.Use + " " + teamsServiceCommandDeleteUseSuffix diff --git a/src/cli/export/export.go b/src/cli/export/export.go index e0deed014..5f63895c0 100644 --- a/src/cli/export/export.go +++ b/src/cli/export/export.go @@ -21,6 +21,8 @@ import ( var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, addSharePointCommands, + addGroupsCommands, + addTeamsCommands, } // AddCommands attaches all `corso export * *` commands to the parent. diff --git a/src/cli/export/groups.go b/src/cli/export/groups.go new file mode 100644 index 000000000..36b56e60f --- /dev/null +++ b/src/cli/export/groups.go @@ -0,0 +1,84 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addGroupsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + groupsServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +// TODO: correct examples +const ( + groupsServiceCommand = "groups" + groupsServiceCommandUseSuffix = " --backup " + + //nolint:lll + groupsServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory +corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory +corso export groups . --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports +corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso export groups [...] ` +func groupsExportCmd() *cobra.Command { + return &cobra.Command{ + Use: groupsServiceCommand, + Short: "Export M365 Groups service data", + RunE: exportGroupsCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing export destination") + } + + return nil + }, + Example: groupsServiceCommandExportExamples, + } +} + +// processes an groups service export. +func exportGroupsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} diff --git a/src/cli/export/groups_test.go b/src/cli/export/groups_test.go new file mode 100644 index 000000000..d2a091e79 --- /dev/null +++ b/src/cli/export/groups_test.go @@ -0,0 +1,94 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type GroupsUnitSuite struct { + tester.Suite +} + +func TestGroupsUnitSuite(t *testing.T) { + suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *GroupsUnitSuite) TestAddGroupsCommands() { + expectUse := groupsServiceCommand + " " + groupsServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export groups", exportCommand, expectUse, groupsExportCmd().Short, exportGroupsCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addGroupsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "groups", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + // assert.NoError(t, err, clues.ToCore(err)) + assert.ErrorIs(t, err, utils.ErrNotYetImplemented, clues.ToCore(err)) + + opts := utils.MakeGroupsOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go index 593149bd9..ea6537dc2 100644 --- a/src/cli/export/onedrive.go +++ b/src/cli/export/onedrive.go @@ -39,7 +39,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { const ( oneDriveServiceCommand = "onedrive" - oneDriveServiceCommandUseSuffix = "--backup " + oneDriveServiceCommandUseSuffix = " --backup " //nolint:lll oneDriveServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory @@ -62,7 +62,7 @@ func oneDriveExportCmd() *cobra.Command { RunE: exportOneDriveCmd, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { - return errors.New("missing restore destination") + return errors.New("missing export destination") } return nil diff --git a/src/cli/export/sharepoint.go b/src/cli/export/sharepoint.go index ec71a5f2b..7293a02f9 100644 --- a/src/cli/export/sharepoint.go +++ b/src/cli/export/sharepoint.go @@ -39,7 +39,7 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { const ( sharePointServiceCommand = "sharepoint" - sharePointServiceCommandUseSuffix = "--backup " + sharePointServiceCommandUseSuffix = " --backup " //nolint:lll sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to my-exports directory @@ -66,7 +66,7 @@ func sharePointExportCmd() *cobra.Command { RunE: exportSharePointCmd, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { - return errors.New("missing restore destination") + return errors.New("missing export destination") } return nil diff --git a/src/cli/export/teams.go b/src/cli/export/teams.go new file mode 100644 index 000000000..7e680c28d --- /dev/null +++ b/src/cli/export/teams.go @@ -0,0 +1,84 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addTeamsCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, teamsExportCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamsServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +// TODO: correct examples +const ( + teamsServiceCommand = "teams" + teamsServiceCommandUseSuffix = " --backup " + + //nolint:lll + teamsServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory +corso export teams my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory +corso export teams . --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports +corso export teams my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso export teams [...] ` +func teamsExportCmd() *cobra.Command { + return &cobra.Command{ + Use: teamsServiceCommand, + Short: "Export M365 Teams service data", + RunE: exportTeamsCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing export destination") + } + + return nil + }, + Example: teamsServiceCommandExportExamples, + } +} + +// processes an teams service export. +func exportTeamsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + return Only(ctx, utils.ErrNotYetImplemented) +} diff --git a/src/cli/export/teams_test.go b/src/cli/export/teams_test.go new file mode 100644 index 000000000..d431359d6 --- /dev/null +++ b/src/cli/export/teams_test.go @@ -0,0 +1,94 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type TeamsUnitSuite struct { + tester.Suite +} + +func TestTeamsUnitSuite(t *testing.T) { + suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *TeamsUnitSuite) TestAddTeamsCommands() { + expectUse := teamsServiceCommand + " " + teamsServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export teams", exportCommand, expectUse, teamsExportCmd().Short, exportTeamsCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addTeamsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "teams", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + // assert.NoError(t, err, clues.ToCore(err)) + assert.ErrorIs(t, err, utils.ErrNotYetImplemented, clues.ToCore(err)) + + opts := utils.MakeTeamsOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/restore/groups.go b/src/cli/restore/groups.go index a98c9d088..3907b17d0 100644 --- a/src/cli/restore/groups.go +++ b/src/cli/restore/groups.go @@ -18,7 +18,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case restoreCommand: - c, fs = utils.AddCommand(cmd, groupsRestoreCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, groupsRestoreCmd(), utils.MarkPreReleaseCommand()) c.Use = c.Use + " " + groupsServiceCommandUseSuffix diff --git a/src/cli/restore/teams.go b/src/cli/restore/teams.go index 59623024a..059c2182a 100644 --- a/src/cli/restore/teams.go +++ b/src/cli/restore/teams.go @@ -18,7 +18,7 @@ func addTeamsCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case restoreCommand: - c, fs = utils.AddCommand(cmd, teamsRestoreCmd(), utils.HideCommand()) + c, fs = utils.AddCommand(cmd, teamsRestoreCmd(), utils.MarkPreReleaseCommand()) c.Use = c.Use + " " + teamsServiceCommandUseSuffix From 90ac62ab140be153616723d0055c7bd8526714b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 20:06:53 +0000 Subject: [PATCH 11/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/go?= =?UTF-8?q?ogle/uuid=20from=201.3.0=20to=201.3.1=20in=20/src=20(#4078)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.3.0 to 1.3.1.
Release notes

Sourced from github.com/google/uuid's releases.

v1.3.1

1.3.1 (2023-08-18)

Bug Fixes

  • Use .EqualFold() to parse urn prefixed UUIDs (#118) (574e687)
Changelog

Sourced from github.com/google/uuid's changelog.

1.3.1 (2023-08-18)

Bug Fixes

  • Use .EqualFold() to parse urn prefixed UUIDs (#118) (574e687)

Changelog

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/google/uuid&package-manager=go_modules&previous-version=1.3.0&new-version=1.3.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index e34158eb9..88e1190fb 100644 --- a/src/go.mod +++ b/src/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.44.327 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/h2non/gock v1.2.0 github.com/kopia/kopia v0.13.0 github.com/microsoft/kiota-abstractions-go v1.2.0 diff --git a/src/go.sum b/src/go.sum index aadbdc6f3..0b2b5786e 100644 --- a/src/go.sum +++ b/src/go.sum @@ -192,8 +192,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= From 6a83d23ff245040864e9a3bbd2e3a51caa02e916 Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 21 Aug 2023 14:44:24 -0600 Subject: [PATCH 12/16] add groups selectors for channels and messages (#4071) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3989 #### Test Plan - [x] :zap: Unit test --- src/pkg/path/category_type.go | 27 +- src/pkg/path/categorytype_string.go | 5 +- src/pkg/selectors/groups.go | 83 +++--- src/pkg/selectors/groups_test.go | 421 ++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+), 47 deletions(-) create mode 100644 src/pkg/selectors/groups_test.go diff --git a/src/pkg/path/category_type.go b/src/pkg/path/category_type.go index 5f8009e5d..40f511692 100644 --- a/src/pkg/path/category_type.go +++ b/src/pkg/path/category_type.go @@ -17,15 +17,16 @@ type CategoryType int //go:generate stringer -type=CategoryType -linecomment const ( - UnknownCategory CategoryType = 0 - EmailCategory CategoryType = 1 // email - ContactsCategory CategoryType = 2 // contacts - EventsCategory CategoryType = 3 // events - FilesCategory CategoryType = 4 // files - ListsCategory CategoryType = 5 // lists - LibrariesCategory CategoryType = 6 // libraries - PagesCategory CategoryType = 7 // pages - DetailsCategory CategoryType = 8 // details + UnknownCategory CategoryType = 0 + EmailCategory CategoryType = 1 // email + ContactsCategory CategoryType = 2 // contacts + EventsCategory CategoryType = 3 // events + FilesCategory CategoryType = 4 // files + ListsCategory CategoryType = 5 // lists + LibrariesCategory CategoryType = 6 // libraries + PagesCategory CategoryType = 7 // pages + DetailsCategory CategoryType = 8 // details + ChannelMessagesCategory CategoryType = 9 // channel messages ) func ToCategoryType(category string) CategoryType { @@ -48,6 +49,8 @@ func ToCategoryType(category string) CategoryType { return PagesCategory case strings.ToLower(DetailsCategory.String()): return DetailsCategory + case strings.ToLower(ChannelMessagesCategory.String()): + return ChannelMessagesCategory default: return UnknownCategory } @@ -73,6 +76,12 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{ ListsCategory: {}, PagesCategory: {}, }, + GroupsService: { + ChannelMessagesCategory: {}, + }, + TeamsService: { + ChannelMessagesCategory: {}, + }, } func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType, error) { diff --git a/src/pkg/path/categorytype_string.go b/src/pkg/path/categorytype_string.go index 626cc4e31..7b548d25a 100644 --- a/src/pkg/path/categorytype_string.go +++ b/src/pkg/path/categorytype_string.go @@ -17,11 +17,12 @@ func _() { _ = x[LibrariesCategory-6] _ = x[PagesCategory-7] _ = x[DetailsCategory-8] + _ = x[ChannelMessagesCategory-9] } -const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetails" +const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannel messages" -var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65} +var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 81} func (i CategoryType) String() string { if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) { diff --git a/src/pkg/selectors/groups.go b/src/pkg/selectors/groups.go index 7adf5398c..30d93698c 100644 --- a/src/pkg/selectors/groups.go +++ b/src/pkg/selectors/groups.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" ) @@ -214,38 +215,42 @@ func (s *groups) AllData() []GroupsScope { scopes = append( scopes, - makeScope[GroupsScope](GroupsTODOContainer, Any())) + makeScope[GroupsScope](GroupsChannel, Any())) return scopes } -// TODO produces one or more Groups TODO scopes. +// Channel produces one or more SharePoint channel scopes, where the channel +// matches upon a given channel by ID or Name. In order to ensure channel selection +// this should always be embedded within the Filter() set; include(channel()) will +// select all items in the channel without further filtering. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] -// Any empty slice defaults to [selectors.None] -func (s *groups) TODO(lists []string, opts ...option) []GroupsScope { +// If any slice is empty, it defaults to [selectors.None] +func (s *groups) Channel(channel string) []GroupsScope { + return []GroupsScope{ + makeInfoScope[GroupsScope]( + GroupsChannel, + GroupsInfoChannel, + []string{channel}, + filters.Equal), + } +} + +// ChannelMessages produces one or more Groups channel message scopes. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (s *sharePoint) ChannelMessages(channels, messages []string, opts ...option) []GroupsScope { var ( scopes = []GroupsScope{} os = append([]option{pathComparator()}, opts...) ) - scopes = append(scopes, makeScope[GroupsScope](GroupsTODOContainer, lists, os...)) - - return scopes -} - -// ListTODOItemsItems produces one or more Groups TODO item scopes. -// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] -// If any slice contains selectors.None, that slice is reduced to [selectors.None] -// If any slice is empty, it defaults to [selectors.None] -// options are only applied to the list scopes. -func (s *groups) TODOItems(lists, items []string, opts ...option) []GroupsScope { - scopes := []GroupsScope{} - scopes = append( scopes, - makeScope[GroupsScope](GroupsTODOItem, items, defaultItemOptions(s.Cfg)...). - set(GroupsTODOContainer, lists, opts...)) + makeScope[GroupsScope](GroupsChannelMessage, messages, os...). + set(GroupsChannel, channels, opts...)) return scopes } @@ -270,21 +275,22 @@ const ( GroupsCategoryUnknown groupsCategory = "" // types of data in Groups - GroupsGroup groupsCategory = "GroupsGroup" - GroupsTODOContainer groupsCategory = "GroupsTODOContainer" - GroupsTODOItem groupsCategory = "GroupsTODOItem" + GroupsGroup groupsCategory = "GroupsGroup" + GroupsChannel groupsCategory = "GroupsChannel" + GroupsChannelMessage groupsCategory = "GroupsChannelMessage" // details.itemInfo comparables - // library drive selection + // channel drive selection GroupsInfoSiteLibraryDrive groupsCategory = "GroupsInfoSiteLibraryDrive" + GroupsInfoChannel groupsCategory = "GroupsInfoChannel" ) // groupsLeafProperties describes common metadata of the leaf categories var groupsLeafProperties = map[categorizer]leafProperty{ - GroupsTODOItem: { // the root category must be represented, even though it isn't a leaf - pathKeys: []categorizer{GroupsTODOContainer, GroupsTODOItem}, - pathType: path.UnknownCategory, + GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf + pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage}, + pathType: path.ChannelMessagesCategory, }, GroupsGroup: { // the root category must be represented, even though it isn't a leaf pathKeys: []categorizer{GroupsGroup}, @@ -303,8 +309,10 @@ func (c groupsCategory) String() string { // Ex: ServiceUser.leafCat() => ServiceUser func (c groupsCategory) leafCat() categorizer { switch c { - case GroupsTODOContainer, GroupsInfoSiteLibraryDrive: - return GroupsTODOItem + // TODO: if channels ever contain more than one type of item, + // we'll need to fix this up. + case GroupsChannel, GroupsChannelMessage, GroupsInfoSiteLibraryDrive: + return GroupsChannelMessage } return c @@ -348,12 +356,12 @@ func (c groupsCategory) pathValues( ) switch c { - case GroupsTODOContainer, GroupsTODOItem: + case GroupsChannel, GroupsChannelMessage: if ent.Groups == nil { return nil, clues.New("no Groups ItemInfo in details") } - folderCat, itemCat = GroupsTODOContainer, GroupsTODOItem + folderCat, itemCat = GroupsChannel, GroupsChannelMessage rFld = ent.Groups.ParentPath default: @@ -451,7 +459,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS os := []option{} switch cat { - case GroupsTODOContainer: + case GroupsChannel: os = append(os, pathComparator()) } @@ -462,10 +470,10 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS func (s GroupsScope) setDefaults() { switch s.Category() { case GroupsGroup: - s[GroupsTODOContainer.String()] = passAny - s[GroupsTODOItem.String()] = passAny - case GroupsTODOContainer: - s[GroupsTODOItem.String()] = passAny + s[GroupsChannel.String()] = passAny + s[GroupsChannelMessage.String()] = passAny + case GroupsChannel: + s[GroupsChannelMessage.String()] = passAny } } @@ -485,7 +493,7 @@ func (s groups) Reduce( deets, s.Selector, map[path.CategoryType]groupsCategory{ - path.UnknownCategory: GroupsTODOItem, + path.ChannelMessagesCategory: GroupsChannelMessage, }, errs) } @@ -516,6 +524,9 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool { } return matchesAny(s, GroupsInfoSiteLibraryDrive, ds) + case GroupsInfoChannel: + ds := Any() + return matchesAny(s, GroupsInfoChannel, ds) } return s.Matches(infoCat, i) diff --git a/src/pkg/selectors/groups_test.go b/src/pkg/selectors/groups_test.go new file mode 100644 index 000000000..a0912a144 --- /dev/null +++ b/src/pkg/selectors/groups_test.go @@ -0,0 +1,421 @@ +package selectors + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/path" +) + +type GroupsSelectorSuite struct { + tester.Suite +} + +func TestGroupsSelectorSuite(t *testing.T) { + suite.Run(t, &GroupsSelectorSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *GroupsSelectorSuite) TestNewGroupsBackup() { + t := suite.T() + ob := NewGroupsBackup(nil) + assert.Equal(t, ob.Service, ServiceGroups) + assert.NotZero(t, ob.Scopes()) +} + +func (suite *GroupsSelectorSuite) TestToGroupsBackup() { + t := suite.T() + ob := NewGroupsBackup(nil) + s := ob.Selector + ob, err := s.ToGroupsBackup() + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, ob.Service, ServiceGroups) + assert.NotZero(t, ob.Scopes()) +} + +func (suite *GroupsSelectorSuite) TestNewGroupsRestore() { + t := suite.T() + or := NewGroupsRestore(nil) + assert.Equal(t, or.Service, ServiceGroups) + assert.NotZero(t, or.Scopes()) +} + +func (suite *GroupsSelectorSuite) TestToGroupsRestore() { + t := suite.T() + eb := NewGroupsRestore(nil) + s := eb.Selector + or, err := s.ToGroupsRestore() + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, or.Service, ServiceGroups) + assert.NotZero(t, or.Scopes()) +} + +// TODO(rkeepers): implement +// func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { +// toRR := func(cat path.CategoryType, siteID string, folders []string, item string) string { +// folderElems := make([]string, 0, len(folders)) + +// for _, f := range folders { +// folderElems = append(folderElems, f+".d") +// } + +// return stubRepoRef( +// path.GroupsService, +// cat, +// siteID, +// strings.Join(folderElems, "/"), +// item) +// } + +// var ( +// prefixElems = []string{ +// odConsts.DrivesPathDir, +// "drive!id", +// odConsts.RootPathDir, +// } +// itemElems1 = []string{"folderA", "folderB"} +// itemElems2 = []string{"folderA", "folderC"} +// itemElems3 = []string{"folderD", "folderE"} +// pairAC = "folderA/folderC" +// pairGH = "folderG/folderH" +// item = toRR( +// path.LibrariesCategory, +// "sid", +// append(slices.Clone(prefixElems), itemElems1...), +// "item") +// item2 = toRR( +// path.LibrariesCategory, +// "sid", +// append(slices.Clone(prefixElems), itemElems2...), +// "item2") +// item3 = toRR( +// path.LibrariesCategory, +// "sid", +// append(slices.Clone(prefixElems), itemElems3...), +// "item3") +// item4 = stubRepoRef(path.GroupsService, path.PagesCategory, "sid", pairGH, "item4") +// item5 = stubRepoRef(path.GroupsService, path.PagesCategory, "sid", pairGH, "item5") +// ) + +// deets := &details.Details{ +// DetailsModel: details.DetailsModel{ +// Entries: []details.Entry{ +// { +// RepoRef: item, +// ItemRef: "item", +// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems1...), "/"), +// ItemInfo: details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsLibrary, +// ItemName: "itemName", +// ParentPath: strings.Join(itemElems1, "/"), +// }, +// }, +// }, +// { +// RepoRef: item2, +// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems2...), "/"), +// // ItemRef intentionally blank to test fallback case +// ItemInfo: details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsLibrary, +// ItemName: "itemName2", +// ParentPath: strings.Join(itemElems2, "/"), +// }, +// }, +// }, +// { +// RepoRef: item3, +// ItemRef: "item3", +// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems3...), "/"), +// ItemInfo: details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsLibrary, +// ItemName: "itemName3", +// ParentPath: strings.Join(itemElems3, "/"), +// }, +// }, +// }, +// { +// RepoRef: item4, +// LocationRef: pairGH, +// ItemRef: "item4", +// ItemInfo: details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsPage, +// ItemName: "itemName4", +// ParentPath: pairGH, +// }, +// }, +// }, +// { +// RepoRef: item5, +// LocationRef: pairGH, +// // ItemRef intentionally blank to test fallback case +// ItemInfo: details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsPage, +// ItemName: "itemName5", +// ParentPath: pairGH, +// }, +// }, +// }, +// }, +// }, +// } + +// arr := func(s ...string) []string { +// return s +// } + +// table := []struct { +// name string +// makeSelector func() *GroupsRestore +// expect []string +// cfg Config +// }{ +// { +// name: "all", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore(Any()) +// odr.Include(odr.AllData()) +// return odr +// }, +// expect: arr(item, item2, item3, item4, item5), +// }, +// { +// name: "only match item", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore(Any()) +// odr.Include(odr.LibraryItems(Any(), []string{"item2"})) +// return odr +// }, +// expect: arr(item2), +// }, +// { +// name: "id doesn't match name", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore(Any()) +// odr.Include(odr.LibraryItems(Any(), []string{"item2"})) +// return odr +// }, +// expect: []string{}, +// cfg: Config{OnlyMatchItemNames: true}, +// }, +// { +// name: "only match item name", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore(Any()) +// odr.Include(odr.LibraryItems(Any(), []string{"itemName2"})) +// return odr +// }, +// expect: arr(item2), +// cfg: Config{OnlyMatchItemNames: true}, +// }, +// { +// name: "name doesn't match", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore(Any()) +// odr.Include(odr.LibraryItems(Any(), []string{"itemName2"})) +// return odr +// }, +// expect: []string{}, +// }, +// { +// name: "only match folder", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore([]string{"sid"}) +// odr.Include(odr.LibraryFolders([]string{"folderA/folderB", pairAC})) +// return odr +// }, +// expect: arr(item, item2), +// }, +// { +// name: "pages match folder", +// makeSelector: func() *GroupsRestore { +// odr := NewGroupsRestore([]string{"sid"}) +// odr.Include(odr.Pages([]string{pairGH, pairAC})) +// return odr +// }, +// expect: arr(item4, item5), +// }, +// } +// for _, test := range table { +// suite.Run(test.name, func() { +// t := suite.T() + +// ctx, flush := tester.NewContext(t) +// defer flush() + +// sel := test.makeSelector() +// sel.Configure(test.cfg) +// results := sel.Reduce(ctx, deets, fault.New(true)) +// paths := results.Paths() +// assert.Equal(t, test.expect, paths) +// }) +// } +// } + +func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() { + var ( + itemName = "item" + itemID = "item-id" + shortRef = "short" + elems = []string{itemID} + ) + + table := []struct { + name string + sc groupsCategory + pathElems []string + locRef string + parentPath string + expected map[categorizer][]string + cfg Config + }{ + { + name: "Groups Channel Messages", + sc: GroupsChannelMessage, + pathElems: elems, + locRef: "", + expected: map[categorizer][]string{ + GroupsChannel: {""}, + GroupsChannelMessage: {itemID, shortRef}, + }, + cfg: Config{}, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + itemPath, err := path.Build( + "tenant", + "site", + path.GroupsService, + test.sc.PathType(), + true, + test.pathElems...) + require.NoError(t, err, clues.ToCore(err)) + + ent := details.Entry{ + RepoRef: itemPath.String(), + ShortRef: shortRef, + ItemRef: itemPath.Item(), + LocationRef: test.locRef, + ItemInfo: details.ItemInfo{ + Groups: &details.GroupsInfo{ + ItemName: itemName, + ParentPath: test.parentPath, + }, + }, + } + + pv, err := test.sc.pathValues(itemPath, ent, test.cfg) + require.NoError(t, err) + assert.Equal(t, test.expected, pv) + }) + } +} + +// TODO(abin): implement +// func (suite *GroupsSelectorSuite) TestGroupsScope_MatchesInfo() { +// var ( +// sel = NewGroupsRestore(Any()) +// host = "www.website.com" +// pth = "/foo" +// url = host + pth +// epoch = time.Time{} +// now = time.Now() +// modification = now.Add(15 * time.Minute) +// future = now.Add(45 * time.Minute) +// ) + +// table := []struct { +// name string +// infoURL string +// scope []GroupsScope +// expect assert.BoolAssertionFunc +// }{ +// {"host match", host, sel.WebURL([]string{host}), assert.True}, +// {"url match", url, sel.WebURL([]string{url}), assert.True}, +// {"host suffixes host", host, sel.WebURL([]string{host}, SuffixMatch()), assert.True}, +// {"url does not suffix host", url, sel.WebURL([]string{host}, SuffixMatch()), assert.False}, +// {"url has path suffix", url, sel.WebURL([]string{pth}, SuffixMatch()), assert.True}, +// {"host does not contain substring", host, sel.WebURL([]string{"website"}), assert.False}, +// {"url does not suffix substring", url, sel.WebURL([]string{"oo"}, SuffixMatch()), assert.False}, +// {"host mismatch", host, sel.WebURL([]string{"www.google.com"}), assert.False}, +// {"file create after the epoch", host, sel.CreatedAfter(dttm.Format(epoch)), assert.True}, +// {"file create after now", host, sel.CreatedAfter(dttm.Format(now)), assert.False}, +// {"file create after later", url, sel.CreatedAfter(dttm.Format(future)), assert.False}, +// {"file create before future", host, sel.CreatedBefore(dttm.Format(future)), assert.True}, +// {"file create before now", host, sel.CreatedBefore(dttm.Format(now)), assert.False}, +// {"file create before modification", host, sel.CreatedBefore(dttm.Format(modification)), assert.True}, +// {"file create before epoch", host, sel.CreatedBefore(dttm.Format(now)), assert.False}, +// {"file modified after the epoch", host, sel.ModifiedAfter(dttm.Format(epoch)), assert.True}, +// {"file modified after now", host, sel.ModifiedAfter(dttm.Format(now)), assert.True}, +// {"file modified after later", host, sel.ModifiedAfter(dttm.Format(future)), assert.False}, +// {"file modified before future", host, sel.ModifiedBefore(dttm.Format(future)), assert.True}, +// {"file modified before now", host, sel.ModifiedBefore(dttm.Format(now)), assert.False}, +// {"file modified before epoch", host, sel.ModifiedBefore(dttm.Format(now)), assert.False}, +// {"in library", host, sel.Library("included-library"), assert.True}, +// {"not in library", host, sel.Library("not-included-library"), assert.False}, +// {"library id", host, sel.Library("1234"), assert.True}, +// {"not library id", host, sel.Library("abcd"), assert.False}, +// } +// for _, test := range table { +// suite.Run(test.name, func() { +// t := suite.T() + +// itemInfo := details.ItemInfo{ +// Groups: &details.GroupsInfo{ +// ItemType: details.GroupsPage, +// WebURL: test.infoURL, +// Created: now, +// Modified: modification, +// DriveName: "included-library", +// DriveID: "1234", +// }, +// } + +// scopes := setScopesToDefault(test.scope) +// for _, scope := range scopes { +// test.expect(t, scope.matchesInfo(itemInfo)) +// } +// }) +// } +// } + +func (suite *GroupsSelectorSuite) TestCategory_PathType() { + table := []struct { + cat groupsCategory + pathType path.CategoryType + }{ + { + cat: GroupsCategoryUnknown, + pathType: path.UnknownCategory, + }, + { + cat: GroupsChannel, + pathType: path.ChannelMessagesCategory, + }, + { + cat: GroupsChannelMessage, + pathType: path.ChannelMessagesCategory, + }, + } + for _, test := range table { + suite.Run(test.cat.String(), func() { + assert.Equal( + suite.T(), + test.pathType.String(), + test.cat.PathType().String()) + }) + } +} From 6963f63f4ff268e572505e072dbe361e444e9bd7 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:40:47 -0700 Subject: [PATCH 13/16] Basic code for backup cleanup (#4051) Starting code for removing item data snapshots, backups, and backup details that have been orphaned. Data can become orphaned through either incomplete backup delete operations (older corso versions) or because backups didn't complete successfully This code doesn't cover all cases (see TODOs in PR) but gets a lot of the boiler-plate that will be required. Future PRs will build on what's in here to close the gaps This code is not wired into any corso operations so it cannot be run outside of unit tests --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3217 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/base_finder.go | 8 - src/internal/kopia/cleanup_backups.go | 156 ++++++++ src/internal/kopia/cleanup_backups_test.go | 433 +++++++++++++++++++++ src/internal/kopia/conn.go | 23 +- 4 files changed, 609 insertions(+), 11 deletions(-) create mode 100644 src/internal/kopia/cleanup_backups.go create mode 100644 src/internal/kopia/cleanup_backups_test.go diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index 83f4009c4..81082ded6 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -115,14 +115,6 @@ func (me ManifestEntry) GetTag(key string) (string, bool) { return v, ok } -type snapshotManager interface { - FindManifests( - ctx context.Context, - tags map[string]string, - ) ([]*manifest.EntryMetadata, error) - LoadSnapshot(ctx context.Context, id manifest.ID) (*snapshot.Manifest, error) -} - func serviceCatString(s path.ServiceType, c path.CategoryType) string { return s.String() + c.String() } diff --git a/src/internal/kopia/cleanup_backups.go b/src/internal/kopia/cleanup_backups.go new file mode 100644 index 000000000..b431b7a91 --- /dev/null +++ b/src/internal/kopia/cleanup_backups.go @@ -0,0 +1,156 @@ +package kopia + +import ( + "context" + "errors" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/snapshot" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/store" +) + +func cleanupOrphanedData( + ctx context.Context, + bs store.Storer, + mf manifestFinder, +) error { + // Get all snapshot manifests. + snaps, err := mf.FindManifests( + ctx, + map[string]string{ + manifest.TypeLabelKey: snapshot.ManifestType, + }) + if err != nil { + return clues.Wrap(err, "getting snapshots") + } + + var ( + // deets is a hash set of the ModelStoreID or snapshot IDs for backup + // details. It contains the IDs for both legacy details stored in the model + // store and newer details stored as a snapshot because it doesn't matter + // what the storage format is. We only need to know the ID so we can: + // 1. check if there's a corresponding backup for them + // 2. delete the details if they're orphaned + deets = map[manifest.ID]struct{}{} + // dataSnaps is a hash set of the snapshot IDs for item data snapshots. + dataSnaps = map[manifest.ID]struct{}{} + ) + + // TODO(ashmrtn): Exclude all snapshots and details younger than X . + // Doing so adds some buffer so that even if this is run concurrently with a + // backup it's not likely to delete models just being created. For example, + // running this when another corso instance has created an item data snapshot + // but hasn't yet created the details snapshot or the backup model would + // result in this instance of corso marking the newly created item data + // snapshot for deletion because it appears orphaned. + // + // Excluding only snapshots and details models works for now since the backup + // model is the last thing persisted out of them. If we switch the order of + // persistence then this will need updated as well. + // + // The buffer duration should be longer than the time it would take to do + // details merging and backup model creation. We don't have hard numbers on + // that, but it should be faster than creating the snapshot itself and + // probably happens O(minutes) or O(hours) instead of O(days). Of course, that + // assumes a non-adversarial setup where things such as machine hiberation, + // process freezing (i.e. paused at the OS level), etc. don't occur. + + // Sort all the snapshots as either details snapshots or item data snapshots. + for _, snap := range snaps { + k, _ := makeTagKV(TagBackupCategory) + if _, ok := snap.Labels[k]; ok { + dataSnaps[snap.ID] = struct{}{} + continue + } + + deets[snap.ID] = struct{}{} + } + + // Get all legacy backup details models. The initial version of backup delete + // didn't seem to delete them so they may also be orphaned if the repo is old + // enough. + deetsModels, err := bs.GetIDsForType(ctx, model.BackupDetailsSchema, nil) + if err != nil { + return clues.Wrap(err, "getting legacy backup details") + } + + for _, d := range deetsModels { + deets[d.ModelStoreID] = struct{}{} + } + + // Get all backup models. + bups, err := bs.GetIDsForType(ctx, model.BackupSchema, nil) + if err != nil { + return clues.Wrap(err, "getting all backup models") + } + + toDelete := maps.Clone(deets) + maps.Copy(toDelete, dataSnaps) + + for _, bup := range bups { + toDelete[manifest.ID(bup.ModelStoreID)] = struct{}{} + + bm := backup.Backup{} + + if err := bs.GetWithModelStoreID( + ctx, + model.BackupSchema, + bup.ModelStoreID, + &bm, + ); err != nil { + if !errors.Is(err, data.ErrNotFound) { + return clues.Wrap(err, "getting backup model"). + With("search_backup_id", bup.ID) + } + + // TODO(ashmrtn): This actually needs revised, see above TODO. Leaving it + // here for the moment to get the basic logic in. + // + // Safe to continue if the model wasn't found because that means that the + // possible item data and details for the backup are now orphaned. They'll + // be deleted since we won't remove them from the delete set. + // + // This isn't expected to really pop up, but it's possible if this + // function is run concurrently with either a backup delete or another + // instance of this function. + logger.Ctx(ctx).Debugw( + "backup model not found", + "search_backup_id", bup.ModelStoreID) + + continue + } + + ssid := bm.StreamStoreID + if len(ssid) == 0 { + ssid = bm.DetailsID + } + + _, dataOK := dataSnaps[manifest.ID(bm.SnapshotID)] + _, deetsOK := deets[manifest.ID(ssid)] + + // All data is present, we shouldn't garbage collect this backup. + if deetsOK && dataOK { + delete(toDelete, bup.ModelStoreID) + delete(toDelete, manifest.ID(bm.SnapshotID)) + delete(toDelete, manifest.ID(ssid)) + } + } + + // Use single atomic batch delete operation to cleanup to keep from making a + // bunch of manifest content blobs. + if err := bs.DeleteWithModelStoreIDs(ctx, maps.Keys(toDelete)...); err != nil { + return clues.Wrap(err, "deleting orphaned data") + } + + // TODO(ashmrtn): Do some pruning of assist backup models so we don't keep + // them around forever. + + return nil +} diff --git a/src/internal/kopia/cleanup_backups_test.go b/src/internal/kopia/cleanup_backups_test.go new file mode 100644 index 000000000..78bc6a164 --- /dev/null +++ b/src/internal/kopia/cleanup_backups_test.go @@ -0,0 +1,433 @@ +package kopia + +import ( + "context" + "fmt" + "testing" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup" +) + +type BackupCleanupUnitSuite struct { + tester.Suite +} + +func TestBackupCleanupUnitSuite(t *testing.T) { + suite.Run(t, &BackupCleanupUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +type mockManifestFinder struct { + t *testing.T + manifests []*manifest.EntryMetadata + err error +} + +func (mmf mockManifestFinder) FindManifests( + ctx context.Context, + tags map[string]string, +) ([]*manifest.EntryMetadata, error) { + assert.Equal( + mmf.t, + map[string]string{"type": "snapshot"}, + tags, + "snapshot search tags") + + return mmf.manifests, clues.Stack(mmf.err).OrNil() +} + +type mockStorer struct { + t *testing.T + + details []*model.BaseModel + detailsErr error + + backups []backupRes + backupListErr error + + expectDeleteIDs []manifest.ID + deleteErr error +} + +func (ms mockStorer) Delete(context.Context, model.Schema, model.StableID) error { + return clues.New("not implemented") +} + +func (ms mockStorer) Get(context.Context, model.Schema, model.StableID, model.Model) error { + return clues.New("not implemented") +} + +func (ms mockStorer) Put(context.Context, model.Schema, model.Model) error { + return clues.New("not implemented") +} + +func (ms mockStorer) Update(context.Context, model.Schema, model.Model) error { + return clues.New("not implemented") +} + +func (ms mockStorer) GetIDsForType( + _ context.Context, + s model.Schema, + tags map[string]string, +) ([]*model.BaseModel, error) { + assert.Empty(ms.t, tags, "model search tags") + + switch s { + case model.BackupDetailsSchema: + return ms.details, clues.Stack(ms.detailsErr).OrNil() + + case model.BackupSchema: + var bases []*model.BaseModel + + for _, b := range ms.backups { + bases = append(bases, &b.bup.BaseModel) + } + + return bases, clues.Stack(ms.backupListErr).OrNil() + } + + return nil, clues.New(fmt.Sprintf("unknown type: %s", s.String())) +} + +func (ms mockStorer) GetWithModelStoreID( + _ context.Context, + s model.Schema, + id manifest.ID, + m model.Model, +) error { + assert.Equal(ms.t, model.BackupSchema, s, "model get schema") + + d := m.(*backup.Backup) + + for _, b := range ms.backups { + if id == b.bup.ModelStoreID { + *d = *b.bup + return clues.Stack(b.err).OrNil() + } + } + + return clues.Stack(data.ErrNotFound) +} + +func (ms mockStorer) DeleteWithModelStoreIDs( + _ context.Context, + ids ...manifest.ID, +) error { + assert.ElementsMatch(ms.t, ms.expectDeleteIDs, ids, "model delete IDs") + return clues.Stack(ms.deleteErr).OrNil() +} + +// backupRes represents an individual return value for an item in GetIDsForType +// or the result of GetWithModelStoreID. err is used for GetWithModelStoreID +// only. +type backupRes struct { + bup *backup.Backup + err error +} + +func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { + backupTag, _ := makeTagKV(TagBackupCategory) + + // Current backup and snapshots. + bupCurrent := &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("current-bup-id"), + ModelStoreID: manifest.ID("current-bup-msid"), + }, + SnapshotID: "current-snap-msid", + StreamStoreID: "current-deets-msid", + } + + snapCurrent := &manifest.EntryMetadata{ + ID: "current-snap-msid", + Labels: map[string]string{ + backupTag: "0", + }, + } + + deetsCurrent := &manifest.EntryMetadata{ + ID: "current-deets-msid", + } + + // Legacy backup with details in separate model. + bupLegacy := &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("legacy-bup-id"), + ModelStoreID: manifest.ID("legacy-bup-msid"), + }, + SnapshotID: "legacy-snap-msid", + DetailsID: "legacy-deets-msid", + } + + snapLegacy := &manifest.EntryMetadata{ + ID: "legacy-snap-msid", + Labels: map[string]string{ + backupTag: "0", + }, + } + + deetsLegacy := &model.BaseModel{ + ID: "legacy-deets-id", + ModelStoreID: "legacy-deets-msid", + } + + // Incomplete backup missing data snapshot. + bupNoSnapshot := &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("ns-bup-id"), + ModelStoreID: manifest.ID("ns-bup-id-msid"), + }, + StreamStoreID: "ns-deets-msid", + } + + deetsNoSnapshot := &manifest.EntryMetadata{ + ID: "ns-deets-msid", + } + + // Legacy incomplete backup missing data snapshot. + bupLegacyNoSnapshot := &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("ns-legacy-bup-id"), + ModelStoreID: manifest.ID("ns-legacy-bup-id-msid"), + }, + DetailsID: "ns-legacy-deets-msid", + } + + deetsLegacyNoSnapshot := &model.BaseModel{ + ID: "ns-legacy-deets-id", + ModelStoreID: "ns-legacy-deets-msid", + } + + // Incomplete backup missing details. + bupNoDetails := &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("nssid-bup-id"), + ModelStoreID: manifest.ID("nssid-bup-msid"), + }, + SnapshotID: "nssid-snap-msid", + } + + snapNoDetails := &manifest.EntryMetadata{ + ID: "nssid-snap-msid", + Labels: map[string]string{ + backupTag: "0", + }, + } + + table := []struct { + name string + snapshots []*manifest.EntryMetadata + snapshotFetchErr error + // only need BaseModel here since we never look inside the details items. + detailsModels []*model.BaseModel + detailsModelListErr error + backups []backupRes + backupListErr error + deleteErr error + + expectDeleteIDs []manifest.ID + expectErr assert.ErrorAssertionFunc + }{ + { + name: "EmptyRepo", + expectErr: assert.NoError, + }, + { + name: "OnlyCompleteBackups Noops", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + deetsCurrent, + snapLegacy, + }, + detailsModels: []*model.BaseModel{ + deetsLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + {bup: bupLegacy}, + }, + expectErr: assert.NoError, + }, + { + name: "MissingFieldsInBackup CausesCleanup", + snapshots: []*manifest.EntryMetadata{ + snapNoDetails, + deetsNoSnapshot, + }, + detailsModels: []*model.BaseModel{ + deetsLegacyNoSnapshot, + }, + backups: []backupRes{ + {bup: bupNoSnapshot}, + {bup: bupLegacyNoSnapshot}, + {bup: bupNoDetails}, + }, + expectDeleteIDs: []manifest.ID{ + manifest.ID(bupNoSnapshot.ModelStoreID), + manifest.ID(bupLegacyNoSnapshot.ModelStoreID), + manifest.ID(bupNoDetails.ModelStoreID), + manifest.ID(deetsLegacyNoSnapshot.ModelStoreID), + snapNoDetails.ID, + deetsNoSnapshot.ID, + }, + expectErr: assert.NoError, + }, + { + name: "MissingSnapshot CausesCleanup", + snapshots: []*manifest.EntryMetadata{ + deetsCurrent, + }, + detailsModels: []*model.BaseModel{ + deetsLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + {bup: bupLegacy}, + }, + expectDeleteIDs: []manifest.ID{ + manifest.ID(bupCurrent.ModelStoreID), + deetsCurrent.ID, + manifest.ID(bupLegacy.ModelStoreID), + manifest.ID(deetsLegacy.ModelStoreID), + }, + expectErr: assert.NoError, + }, + { + name: "MissingDetails CausesCleanup", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + snapLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + {bup: bupLegacy}, + }, + expectDeleteIDs: []manifest.ID{ + manifest.ID(bupCurrent.ModelStoreID), + manifest.ID(bupLegacy.ModelStoreID), + snapCurrent.ID, + snapLegacy.ID, + }, + expectErr: assert.NoError, + }, + { + name: "SnapshotsListError Fails", + snapshotFetchErr: assert.AnError, + backups: []backupRes{ + {bup: bupCurrent}, + }, + expectErr: assert.Error, + }, + { + name: "LegacyDetailsListError Fails", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + }, + detailsModelListErr: assert.AnError, + backups: []backupRes{ + {bup: bupCurrent}, + }, + expectErr: assert.Error, + }, + { + name: "BackupIDsListError Fails", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + deetsCurrent, + }, + backupListErr: assert.AnError, + expectErr: assert.Error, + }, + { + name: "BackupModelGetErrorNotFound CausesCleanup", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + deetsCurrent, + snapLegacy, + snapNoDetails, + }, + detailsModels: []*model.BaseModel{ + deetsLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + { + bup: bupLegacy, + err: data.ErrNotFound, + }, + { + bup: bupNoDetails, + err: data.ErrNotFound, + }, + }, + // Backup IDs are still included in here because they're added to the + // deletion set prior to attempting to fetch models. The model store + // delete operation should ignore missing models though so there's no + // issue. + expectDeleteIDs: []manifest.ID{ + snapLegacy.ID, + manifest.ID(deetsLegacy.ModelStoreID), + manifest.ID(bupLegacy.ModelStoreID), + snapNoDetails.ID, + manifest.ID(bupNoDetails.ModelStoreID), + }, + expectErr: assert.NoError, + }, + { + name: "BackupModelGetError Fails", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + deetsCurrent, + snapLegacy, + snapNoDetails, + }, + detailsModels: []*model.BaseModel{ + deetsLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + { + bup: bupLegacy, + err: assert.AnError, + }, + {bup: bupNoDetails}, + }, + expectErr: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mbs := mockStorer{ + t: t, + details: test.detailsModels, + detailsErr: test.detailsModelListErr, + backups: test.backups, + backupListErr: test.backupListErr, + expectDeleteIDs: test.expectDeleteIDs, + deleteErr: test.deleteErr, + } + + mmf := mockManifestFinder{ + t: t, + manifests: test.snapshots, + err: test.snapshotFetchErr, + } + + err := cleanupOrphanedData(ctx, mbs, mmf) + test.expectErr(t, err, clues.ToCore(err)) + }) + } +} diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index 7eac9df5c..1703b466d 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -52,9 +52,26 @@ var ( } ) -type snapshotLoader interface { - SnapshotRoot(man *snapshot.Manifest) (fs.Entry, error) -} +type ( + manifestFinder interface { + FindManifests( + ctx context.Context, + tags map[string]string, + ) ([]*manifest.EntryMetadata, error) + } + + snapshotManager interface { + manifestFinder + LoadSnapshot( + ctx context.Context, + id manifest.ID, + ) (*snapshot.Manifest, error) + } + + snapshotLoader interface { + SnapshotRoot(man *snapshot.Manifest) (fs.Entry, error) + } +) var ( _ snapshotManager = &conn{} From 5808797fc64385a38059c8b07adbbbcf9f115021 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 23:11:02 +0000 Subject: [PATCH 14/16] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.327=20to=201.44.328=20in=20/src=20(#?= =?UTF-8?q?4079)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.327 to 1.44.328.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.328 (2023-08-21)

Service Client Updates

  • service/cloud9: Adds new service
    • Doc only update to add Ubuntu 22.04 as an Image ID option for Cloud9
  • service/ec2: Updates service API and documentation
    • The DeleteKeyPair API has been updated to return the keyPairId when an existing key pair is deleted.
  • service/finspace: Updates service API and documentation
  • service/rds: Updates service API, documentation, waiters, paginators, and examples
    • Adding support for RDS Aurora Global Database Unplanned Failover
  • service/route53domains: Updates service documentation
    • Fixed typos in description fields
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.327&new-version=1.44.328)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 88e1190fb..8438cb70f 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.327 + github.com/aws/aws-sdk-go v1.44.328 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.1 diff --git a/src/go.sum b/src/go.sum index 0b2b5786e..b88a52f28 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= -github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.328 h1:WBwlf8ym9SDQ/GTIBO9eXyvwappKJyOetWJKl4mT7ZU= +github.com/aws/aws-sdk-go v1.44.328/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 99edf7d5b0a3393dd2824b28ca24118d52d6f8ed Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:48:57 -0700 Subject: [PATCH 15/16] Add and populate mod time for BaseModel (#4065) Get the last time a model was modified and return it in BaseModel. This will help with discovering what items can be garbage collected during incomplete backup cleanup as we don't want to accidentally delete in-flight backups. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3217 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/model_store.go | 1 + src/internal/kopia/model_store_test.go | 48 +++++++++++++++++++++----- src/internal/model/model.go | 5 ++- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/internal/kopia/model_store.go b/src/internal/kopia/model_store.go index 93ef6c182..b5e572844 100644 --- a/src/internal/kopia/model_store.go +++ b/src/internal/kopia/model_store.go @@ -210,6 +210,7 @@ func (ms ModelStore) populateBaseModelFromMetadata( base.ID = model.StableID(id) base.ModelVersion = v base.Tags = m.Labels + base.ModTime = m.ModTime stripHiddenTags(base.Tags) diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index 048817a54..0afc72a7c 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -4,6 +4,7 @@ import ( "context" "sync" "testing" + "time" "github.com/alcionai/clues" "github.com/google/uuid" @@ -34,6 +35,18 @@ func getModelStore(t *testing.T, ctx context.Context) *ModelStore { return &ModelStore{c: c, modelVersion: globalModelVersion} } +func assertEqualNoModTime(t *testing.T, expected, got *fooModel) { + t.Helper() + + expectedClean := *expected + gotClean := *got + + expectedClean.ModTime = time.Time{} + gotClean.ModTime = time.Time{} + + assert.Equal(t, expectedClean, gotClean) +} + // --------------- // unit tests // --------------- @@ -259,6 +272,8 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet() { // Avoid some silly test errors from comparing nil to empty map. foo.Tags = map[string]string{} + startTime := time.Now() + err := suite.m.Put(suite.ctx, test.s, foo) test.check(t, err, clues.ToCore(err)) @@ -273,11 +288,17 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet() { returned := &fooModel{} err = suite.m.Get(suite.ctx, test.s, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + + assertEqualNoModTime(t, foo, returned) + assert.WithinDuration(t, startTime, returned.ModTime, 5*time.Second) + + returned = &fooModel{} err = suite.m.GetWithModelStoreID(suite.ctx, test.s, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + + assertEqualNoModTime(t, foo, returned) + assert.WithinDuration(t, startTime, returned.ModTime, 5*time.Second) }) } } @@ -324,11 +345,11 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet_PreSetID() { err = suite.m.Get(suite.ctx, mdl, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) err = suite.m.GetWithModelStoreID(suite.ctx, mdl, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) }) } } @@ -350,11 +371,11 @@ func (suite *ModelStoreIntegrationSuite) TestPutGet_WithTags() { returned := &fooModel{} err = suite.m.Get(suite.ctx, theModelType, foo.ID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) err = suite.m.GetWithModelStoreID(suite.ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) } func (suite *ModelStoreIntegrationSuite) TestGet_NotFoundErrors() { @@ -559,7 +580,16 @@ func (suite *ModelStoreIntegrationSuite) TestGetOfTypeWithTags() { ids, err := suite.m.GetIDsForType(suite.ctx, test.s, test.tags) require.NoError(t, err, clues.ToCore(err)) - assert.ElementsMatch(t, expected, ids) + cleanIDs := make([]*model.BaseModel, 0, len(ids)) + + for _, id := range ids { + id2 := *id + id2.ModTime = time.Time{} + + cleanIDs = append(cleanIDs, &id2) + } + + assert.ElementsMatch(t, expected, cleanIDs) }) } } @@ -627,7 +657,7 @@ func (suite *ModelStoreIntegrationSuite) TestPutUpdate() { err = m.GetWithModelStoreID(ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) ids, err := m.GetIDsForType(ctx, theModelType, nil) require.NoError(t, err, clues.ToCore(err)) @@ -822,7 +852,7 @@ func (suite *ModelStoreRegressionSuite) TestFailDuringWriteSessionHasNoVisibleEf err = m.GetWithModelStoreID(ctx, theModelType, foo.ModelStoreID, returned) require.NoError(t, err, clues.ToCore(err)) - assert.Equal(t, foo, returned) + assertEqualNoModTime(t, foo, returned) } func openConnAndModelStore( diff --git a/src/internal/model/model.go b/src/internal/model/model.go index a3f25c820..fb72e3613 100644 --- a/src/internal/model/model.go +++ b/src/internal/model/model.go @@ -1,6 +1,8 @@ package model import ( + "time" + "github.com/kopia/kopia/repo/manifest" ) @@ -68,7 +70,8 @@ type BaseModel struct { // Tags associated with this model in the store to facilitate lookup. Tags in // the struct are not serialized directly into the stored model, but are part // of the metadata for the model. - Tags map[string]string `json:"-"` + Tags map[string]string `json:"-"` + ModTime time.Time `json:"-"` } func (bm *BaseModel) Base() *BaseModel { From 11253bf8164025bf32783b9a412f1515f42eed2f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:39:24 -0700 Subject: [PATCH 16/16] Exclude recently created models from garbage collection (#4066) Exclude models that have been created within the buffer period from garbage collection/orphaned checks so that we don't accidentally delete models for backups that are running concurrently with the garbage collection task --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3217 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/cleanup_backups.go | 81 ++++++++++----- src/internal/kopia/cleanup_backups_test.go | 109 ++++++++++++++++++++- 2 files changed, 166 insertions(+), 24 deletions(-) diff --git a/src/internal/kopia/cleanup_backups.go b/src/internal/kopia/cleanup_backups.go index b431b7a91..82ae04dc4 100644 --- a/src/internal/kopia/cleanup_backups.go +++ b/src/internal/kopia/cleanup_backups.go @@ -3,6 +3,7 @@ package kopia import ( "context" "errors" + "time" "github.com/alcionai/clues" "github.com/kopia/kopia/repo/manifest" @@ -16,10 +17,37 @@ import ( "github.com/alcionai/corso/src/pkg/store" ) +// cleanupOrphanedData uses bs and mf to lookup all models/snapshots for backups +// and deletes items that are older than nowFunc() - gcBuffer (cutoff) that are +// not "complete" backups with: +// - a backup model +// - an item data snapshot +// - a details snapshot or details model +// +// We exclude all items younger than the cutoff to add some buffer so that even +// if this is run concurrently with a backup it's not likely to delete models +// just being created. For example, if there was no buffer period and this is +// run when another corso instance has created an item data snapshot but hasn't +// yet created the details snapshot or the backup model it would result in this +// instance of corso marking the newly created item data snapshot for deletion +// because it appears orphaned. +// +// The buffer duration should be longer than the difference in creation times +// between the first item data snapshot/details/backup model made during a +// backup operation and the last. +// +// We don't have hard numbers on the time right now, but if the order of +// persistence is (item data snapshot, details snapshot, backup model) it should +// be faster than creating the snapshot itself and probably happens O(minutes) +// or O(hours) instead of O(days). Of course, that assumes a non-adversarial +// setup where things such as machine hiberation, process freezing (i.e. paused +// at the OS level), etc. don't occur. func cleanupOrphanedData( ctx context.Context, bs store.Storer, mf manifestFinder, + gcBuffer time.Duration, + nowFunc func() time.Time, ) error { // Get all snapshot manifests. snaps, err := mf.FindManifests( @@ -43,27 +71,16 @@ func cleanupOrphanedData( dataSnaps = map[manifest.ID]struct{}{} ) - // TODO(ashmrtn): Exclude all snapshots and details younger than X . - // Doing so adds some buffer so that even if this is run concurrently with a - // backup it's not likely to delete models just being created. For example, - // running this when another corso instance has created an item data snapshot - // but hasn't yet created the details snapshot or the backup model would - // result in this instance of corso marking the newly created item data - // snapshot for deletion because it appears orphaned. - // - // Excluding only snapshots and details models works for now since the backup - // model is the last thing persisted out of them. If we switch the order of - // persistence then this will need updated as well. - // - // The buffer duration should be longer than the time it would take to do - // details merging and backup model creation. We don't have hard numbers on - // that, but it should be faster than creating the snapshot itself and - // probably happens O(minutes) or O(hours) instead of O(days). Of course, that - // assumes a non-adversarial setup where things such as machine hiberation, - // process freezing (i.e. paused at the OS level), etc. don't occur. + cutoff := nowFunc().Add(-gcBuffer) // Sort all the snapshots as either details snapshots or item data snapshots. for _, snap := range snaps { + // Don't even try to see if this needs garbage collected because it's not + // old enough and may correspond to an in-progress operation. + if !cutoff.After(snap.ModTime) { + continue + } + k, _ := makeTagKV(TagBackupCategory) if _, ok := snap.Labels[k]; ok { dataSnaps[snap.ID] = struct{}{} @@ -82,6 +99,12 @@ func cleanupOrphanedData( } for _, d := range deetsModels { + // Don't even try to see if this needs garbage collected because it's not + // old enough and may correspond to an in-progress operation. + if !cutoff.After(d.ModTime) { + continue + } + deets[d.ModelStoreID] = struct{}{} } @@ -95,6 +118,12 @@ func cleanupOrphanedData( maps.Copy(toDelete, dataSnaps) for _, bup := range bups { + // Don't even try to see if this needs garbage collected because it's not + // old enough and may correspond to an in-progress operation. + if !cutoff.After(bup.ModTime) { + continue + } + toDelete[manifest.ID(bup.ModelStoreID)] = struct{}{} bm := backup.Backup{} @@ -110,12 +139,13 @@ func cleanupOrphanedData( With("search_backup_id", bup.ID) } - // TODO(ashmrtn): This actually needs revised, see above TODO. Leaving it - // here for the moment to get the basic logic in. + // Probably safe to continue if the model wasn't found because that means + // that the possible item data and details for the backup are now + // orphaned. They'll be deleted since we won't remove them from the delete + // set. // - // Safe to continue if the model wasn't found because that means that the - // possible item data and details for the backup are now orphaned. They'll - // be deleted since we won't remove them from the delete set. + // The fact that we exclude all items younger than the cutoff should + // already exclude items that are from concurrent corso backup operations. // // This isn't expected to really pop up, but it's possible if this // function is run concurrently with either a backup delete or another @@ -143,6 +173,11 @@ func cleanupOrphanedData( } } + logger.Ctx(ctx).Infow( + "garbage collecting orphaned items", + "num_items", len(toDelete), + "kopia_ids", maps.Keys(toDelete)) + // Use single atomic batch delete operation to cleanup to keep from making a // bunch of manifest content blobs. if err := bs.DeleteWithModelStoreIDs(ctx, maps.Keys(toDelete)...); err != nil { diff --git a/src/internal/kopia/cleanup_backups_test.go b/src/internal/kopia/cleanup_backups_test.go index 78bc6a164..895d9226e 100644 --- a/src/internal/kopia/cleanup_backups_test.go +++ b/src/internal/kopia/cleanup_backups_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/alcionai/clues" "github.com/kopia/kopia/repo/manifest" @@ -221,6 +222,28 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { }, } + // Get some stable time so that we can do everything relative to this in the + // tests. Mostly just makes reasoning/viewing times easier because the only + // differences will be the changes we make. + baseTime := time.Now() + + manifestWithTime := func( + mt time.Time, + m *manifest.EntryMetadata, + ) *manifest.EntryMetadata { + res := *m + res.ModTime = mt + + return &res + } + + backupWithTime := func(mt time.Time, b *backup.Backup) *backup.Backup { + res := *b + res.ModTime = mt + + return &res + } + table := []struct { name string snapshots []*manifest.EntryMetadata @@ -231,12 +254,15 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { backups []backupRes backupListErr error deleteErr error + time time.Time + buffer time.Duration expectDeleteIDs []manifest.ID expectErr assert.ErrorAssertionFunc }{ { name: "EmptyRepo", + time: baseTime, expectErr: assert.NoError, }, { @@ -253,6 +279,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { {bup: bupCurrent}, {bup: bupLegacy}, }, + time: baseTime, expectErr: assert.NoError, }, { @@ -277,6 +304,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { snapNoDetails.ID, deetsNoSnapshot.ID, }, + time: baseTime, expectErr: assert.NoError, }, { @@ -297,6 +325,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { manifest.ID(bupLegacy.ModelStoreID), manifest.ID(deetsLegacy.ModelStoreID), }, + time: baseTime, expectErr: assert.NoError, }, { @@ -315,6 +344,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { snapCurrent.ID, snapLegacy.ID, }, + time: baseTime, expectErr: assert.NoError, }, { @@ -334,6 +364,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { backups: []backupRes{ {bup: bupCurrent}, }, + time: baseTime, expectErr: assert.Error, }, { @@ -343,6 +374,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { deetsCurrent, }, backupListErr: assert.AnError, + time: baseTime, expectErr: assert.Error, }, { @@ -378,6 +410,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { snapNoDetails.ID, manifest.ID(bupNoDetails.ModelStoreID), }, + time: baseTime, expectErr: assert.NoError, }, { @@ -399,8 +432,77 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { }, {bup: bupNoDetails}, }, + time: baseTime, expectErr: assert.Error, }, + { + name: "DeleteError Fails", + snapshots: []*manifest.EntryMetadata{ + snapCurrent, + deetsCurrent, + snapLegacy, + snapNoDetails, + }, + detailsModels: []*model.BaseModel{ + deetsLegacy, + }, + backups: []backupRes{ + {bup: bupCurrent}, + {bup: bupLegacy}, + {bup: bupNoDetails}, + }, + expectDeleteIDs: []manifest.ID{ + snapNoDetails.ID, + manifest.ID(bupNoDetails.ModelStoreID), + }, + deleteErr: assert.AnError, + time: baseTime, + expectErr: assert.Error, + }, + { + name: "MissingSnapshot BarelyTooYoungForCleanup Noops", + snapshots: []*manifest.EntryMetadata{ + manifestWithTime(baseTime, deetsCurrent), + }, + backups: []backupRes{ + {bup: backupWithTime(baseTime, bupCurrent)}, + }, + time: baseTime.Add(24 * time.Hour), + buffer: 24 * time.Hour, + expectErr: assert.NoError, + }, + { + name: "MissingSnapshot BarelyOldEnough CausesCleanup", + snapshots: []*manifest.EntryMetadata{ + manifestWithTime(baseTime, deetsCurrent), + }, + backups: []backupRes{ + {bup: backupWithTime(baseTime, bupCurrent)}, + }, + expectDeleteIDs: []manifest.ID{ + deetsCurrent.ID, + manifest.ID(bupCurrent.ModelStoreID), + }, + time: baseTime.Add((24 * time.Hour) + time.Second), + buffer: 24 * time.Hour, + expectErr: assert.NoError, + }, + { + name: "BackupGetErrorNotFound TooYoung Noops", + snapshots: []*manifest.EntryMetadata{ + manifestWithTime(baseTime, snapCurrent), + manifestWithTime(baseTime, deetsCurrent), + }, + backups: []backupRes{ + { + bup: backupWithTime(baseTime, bupCurrent), + err: data.ErrNotFound, + }, + }, + time: baseTime, + buffer: 24 * time.Hour, + expectErr: assert.NoError, + }, } for _, test := range table { @@ -426,7 +528,12 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() { err: test.snapshotFetchErr, } - err := cleanupOrphanedData(ctx, mbs, mmf) + err := cleanupOrphanedData( + ctx, + mbs, + mmf, + test.buffer, + func() time.Time { return test.time }) test.expectErr(t, err, clues.ToCore(err)) }) }