use channels api to run backups (#4136)
This is the first in a series of PRs to get v0 backups working for channels. In this change, the current api enumerators get plugged into the collections handler to produce backup data. Follow-up PRs will: * hook backup to the CLI * swap full-item enumeration for id-first-get-later pattern * populate each message with all its replies on the get-later * turn on integration testing at the operations and ci layers --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🌻 Feature #### Issue(s) * #3989 #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
ccdb672026
commit
5aee3cc2ae
@ -17,7 +17,6 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO: incremental support
|
// TODO: incremental support
|
||||||
@ -31,7 +30,7 @@ import (
|
|||||||
func CreateCollections(
|
func CreateCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
bpc inject.BackupProducerConfig,
|
bpc inject.BackupProducerConfig,
|
||||||
handler BackupHandler,
|
bh backupHandler,
|
||||||
tenantID string,
|
tenantID string,
|
||||||
scope selectors.GroupsScope,
|
scope selectors.GroupsScope,
|
||||||
// dps DeltaPaths,
|
// dps DeltaPaths,
|
||||||
@ -55,19 +54,15 @@ func CreateCollections(
|
|||||||
observe.Bulletf("%s", qp.Category))
|
observe.Bulletf("%s", qp.Category))
|
||||||
defer close(catProgress)
|
defer close(catProgress)
|
||||||
|
|
||||||
// TODO(keepers): probably shouldn't call out channels here specifically.
|
channels, err := bh.getChannels(ctx)
|
||||||
// This should be a generic container handler. But we don't need
|
if err != nil {
|
||||||
// to worry about that until if/when we use this code to get email
|
return nil, clues.Stack(err)
|
||||||
// conversations as well.
|
}
|
||||||
// Also, this should be produced by the Handler.
|
|
||||||
// chanPager := handler.NewChannelsPager(qp.ProtectedResource.ID())
|
|
||||||
// TODO(neha): enumerate channels
|
|
||||||
channels := []graph.Displayable{}
|
|
||||||
|
|
||||||
collections, err := populateCollections(
|
collections, err := populateCollections(
|
||||||
ctx,
|
ctx,
|
||||||
qp,
|
qp,
|
||||||
handler,
|
bh,
|
||||||
su,
|
su,
|
||||||
channels,
|
channels,
|
||||||
scope,
|
scope,
|
||||||
@ -88,9 +83,9 @@ func CreateCollections(
|
|||||||
func populateCollections(
|
func populateCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
bh BackupHandler,
|
bh backupHandler,
|
||||||
statusUpdater support.StatusUpdater,
|
statusUpdater support.StatusUpdater,
|
||||||
channels []graph.Displayable,
|
channels []models.Channelable,
|
||||||
scope selectors.GroupsScope,
|
scope selectors.GroupsScope,
|
||||||
// dps DeltaPaths,
|
// dps DeltaPaths,
|
||||||
ctrlOpts control.Options,
|
ctrlOpts control.Options,
|
||||||
@ -100,7 +95,6 @@ func populateCollections(
|
|||||||
channelCollections := map[string]data.BackupCollection{}
|
channelCollections := map[string]data.BackupCollection{}
|
||||||
|
|
||||||
// channel ID -> delta url or folder path lookups
|
// channel ID -> delta url or folder path lookups
|
||||||
// TODO(neha/keepers): figure out if deltas are stored per channel, or per group.
|
|
||||||
// deltaURLs = map[string]string{}
|
// deltaURLs = map[string]string{}
|
||||||
// currPaths = map[string]string{}
|
// currPaths = map[string]string{}
|
||||||
// copy of previousPaths. every channel present in the slice param
|
// copy of previousPaths. every channel present in the slice param
|
||||||
@ -118,11 +112,12 @@ func populateCollections(
|
|||||||
return nil, el.Failure()
|
return nil, el.Failure()
|
||||||
}
|
}
|
||||||
|
|
||||||
cID := ptr.Val(c.GetId())
|
|
||||||
// delete(tombstones, cID)
|
// delete(tombstones, cID)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
err error
|
cID = ptr.Val(c.GetId())
|
||||||
|
cName = ptr.Val(c.GetDisplayName())
|
||||||
|
err error
|
||||||
// dp = dps[cID]
|
// dp = dps[cID]
|
||||||
// prevDelta = dp.Delta
|
// prevDelta = dp.Delta
|
||||||
// prevPathStr = dp.Path // do not log: pii; log prevPath instead
|
// prevPathStr = dp.Path // do not log: pii; log prevPath instead
|
||||||
@ -137,10 +132,8 @@ func populateCollections(
|
|||||||
// })
|
// })
|
||||||
)
|
)
|
||||||
|
|
||||||
// currPath, locPath
|
|
||||||
// TODO(rkeepers): the handler should provide this functionality.
|
|
||||||
// Only create a collection if the path matches the scope.
|
// Only create a collection if the path matches the scope.
|
||||||
if !includeContainer(ictx, qp, c, scope, qp.Category) {
|
if !bh.includeContainer(ictx, qp, c, scope) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -154,10 +147,7 @@ func populateCollections(
|
|||||||
|
|
||||||
// ictx = clues.Add(ictx, "previous_path", prevPath)
|
// ictx = clues.Add(ictx, "previous_path", prevPath)
|
||||||
|
|
||||||
// TODO: the handler should provide this implementation.
|
items, _, err := bh.getChannelMessagesDelta(ctx, cID, "")
|
||||||
items, err := collectItems(
|
|
||||||
ctx,
|
|
||||||
bh.NewMessagePager(qp.ProtectedResource.ID(), ptr.Val(c.GetId())))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
el.AddRecoverable(ctx, clues.Stack(err))
|
el.AddRecoverable(ctx, clues.Stack(err))
|
||||||
continue
|
continue
|
||||||
@ -171,15 +161,7 @@ func populateCollections(
|
|||||||
|
|
||||||
var prevPath path.Path
|
var prevPath path.Path
|
||||||
|
|
||||||
// TODO: retrieve from handler
|
currPath, err := bh.canonicalPath(path.Builder{}.Append(cID), qp.TenantID)
|
||||||
currPath, err := path.Builder{}.
|
|
||||||
Append(ptr.Val(c.GetId())).
|
|
||||||
ToDataLayerPath(
|
|
||||||
qp.TenantID,
|
|
||||||
qp.ProtectedResource.ID(),
|
|
||||||
path.GroupsService,
|
|
||||||
qp.Category,
|
|
||||||
true)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
el.AddRecoverable(ctx, clues.Stack(err))
|
el.AddRecoverable(ctx, clues.Stack(err))
|
||||||
continue
|
continue
|
||||||
@ -189,7 +171,7 @@ func populateCollections(
|
|||||||
qp.ProtectedResource.ID(),
|
qp.ProtectedResource.ID(),
|
||||||
currPath,
|
currPath,
|
||||||
prevPath,
|
prevPath,
|
||||||
path.Builder{}.Append(ptr.Val(c.GetDisplayName())),
|
path.Builder{}.Append(cName),
|
||||||
qp.Category,
|
qp.Category,
|
||||||
statusUpdater,
|
statusUpdater,
|
||||||
ctrlOpts)
|
ctrlOpts)
|
||||||
@ -209,7 +191,7 @@ func populateCollections(
|
|||||||
// // as the "previous path", for reference in case of a rename or relocation.
|
// // as the "previous path", for reference in case of a rename or relocation.
|
||||||
// currPaths[cID] = currPath.String()
|
// currPaths[cID] = currPath.String()
|
||||||
|
|
||||||
// FIXME: normally this goes before removal, but linters
|
// FIXME: normally this goes before removal, but the linters require no bottom comments
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
edc.added[ptr.Val(item.GetId())] = struct{}{}
|
edc.added[ptr.Val(item.GetId())] = struct{}{}
|
||||||
}
|
}
|
||||||
@ -240,79 +222,3 @@ func populateCollections(
|
|||||||
|
|
||||||
return channelCollections, el.Failure()
|
return channelCollections, el.Failure()
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectItems(
|
|
||||||
ctx context.Context,
|
|
||||||
pager api.DeltaPager[models.ChatMessageable],
|
|
||||||
) ([]models.ChatMessageable, error) {
|
|
||||||
items := []models.ChatMessageable{}
|
|
||||||
|
|
||||||
for {
|
|
||||||
// assume delta urls here, which allows single-token consumption
|
|
||||||
page, err := pager.GetPage(graph.ConsumeNTokens(ctx, graph.SingleGetOrDeltaLC))
|
|
||||||
if err != nil {
|
|
||||||
return nil, graph.Wrap(ctx, err, "getting page")
|
|
||||||
}
|
|
||||||
|
|
||||||
// if graph.IsErrInvalidDelta(err) {
|
|
||||||
// logger.Ctx(ctx).Infow("Invalid previous delta link", "link", prevDelta)
|
|
||||||
|
|
||||||
// invalidPrevDelta = true
|
|
||||||
// newPaths = map[string]string{}
|
|
||||||
|
|
||||||
// pager.Reset()
|
|
||||||
|
|
||||||
// continue
|
|
||||||
// }
|
|
||||||
|
|
||||||
vals, err := pager.ValuesIn(page)
|
|
||||||
if err != nil {
|
|
||||||
return nil, graph.Wrap(ctx, err, "getting items in page")
|
|
||||||
}
|
|
||||||
|
|
||||||
items = append(items, vals...)
|
|
||||||
|
|
||||||
nextLink, _ := api.NextAndDeltaLink(page)
|
|
||||||
|
|
||||||
// if len(deltaLink) > 0 {
|
|
||||||
// newDeltaURL = deltaLink
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Check if there are more items
|
|
||||||
if len(nextLink) == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Ctx(ctx).Debugw("found nextLink", "next_link", nextLink)
|
|
||||||
pager.SetNext(nextLink)
|
|
||||||
}
|
|
||||||
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns true if the container passes the scope comparison and should be included.
|
|
||||||
// Returns:
|
|
||||||
// - the path representing the directory as it should be stored in the repository.
|
|
||||||
// - the human-readable path using display names.
|
|
||||||
// - true if the path passes the scope comparison.
|
|
||||||
func includeContainer(
|
|
||||||
ctx context.Context,
|
|
||||||
qp graph.QueryParams,
|
|
||||||
gd graph.Displayable,
|
|
||||||
scope selectors.GroupsScope,
|
|
||||||
category path.CategoryType,
|
|
||||||
) bool {
|
|
||||||
// assume a single-level hierarchy
|
|
||||||
directory := ptr.Val(gd.GetDisplayName())
|
|
||||||
|
|
||||||
// TODO(keepers): awaiting parent branch to update to main
|
|
||||||
ok := scope.Matches(selectors.GroupsCategoryUnknown, directory)
|
|
||||||
|
|
||||||
logger.Ctx(ctx).With(
|
|
||||||
"included", ok,
|
|
||||||
"scope", scope,
|
|
||||||
"match_target", directory,
|
|
||||||
).Debug("backup folder selection filter")
|
|
||||||
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|||||||
392
src/internal/m365/collection/groups/backup_test.go
Normal file
392
src/internal/m365/collection/groups/backup_test.go
Normal file
@ -0,0 +1,392 @@
|
|||||||
|
package groups
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/alcionai/clues"
|
||||||
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
||||||
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/collection/groups/testdata"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/support"
|
||||||
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||||
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
|
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// mocks
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
var _ backupHandler = &mockBackupHandler{}
|
||||||
|
|
||||||
|
type mockBackupHandler struct {
|
||||||
|
channels []models.Channelable
|
||||||
|
channelsErr error
|
||||||
|
messages []models.ChatMessageable
|
||||||
|
messagesErr error
|
||||||
|
doNotInclude bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh mockBackupHandler) getChannels(context.Context) ([]models.Channelable, error) {
|
||||||
|
return bh.channels, bh.channelsErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh mockBackupHandler) getChannelMessagesDelta(
|
||||||
|
_ context.Context,
|
||||||
|
_, _ string,
|
||||||
|
) ([]models.ChatMessageable, api.DeltaUpdate, error) {
|
||||||
|
return bh.messages, api.DeltaUpdate{}, bh.messagesErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh mockBackupHandler) includeContainer(
|
||||||
|
context.Context,
|
||||||
|
graph.QueryParams,
|
||||||
|
models.Channelable,
|
||||||
|
selectors.GroupsScope,
|
||||||
|
) bool {
|
||||||
|
return !bh.doNotInclude
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh mockBackupHandler) canonicalPath(
|
||||||
|
folders *path.Builder,
|
||||||
|
tenantID string,
|
||||||
|
) (path.Path, error) {
|
||||||
|
return folders.
|
||||||
|
ToDataLayerPath(
|
||||||
|
tenantID,
|
||||||
|
"protectedResource",
|
||||||
|
path.GroupsService,
|
||||||
|
path.ChannelMessagesCategory,
|
||||||
|
false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Unit Suite
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type BackupUnitSuite struct {
|
||||||
|
tester.Suite
|
||||||
|
creds account.M365Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServiceIteratorsUnitSuite(t *testing.T) {
|
||||||
|
suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *BackupUnitSuite) SetupSuite() {
|
||||||
|
a := tconfig.NewFakeM365Account(suite.T())
|
||||||
|
m365, err := a.M365Config()
|
||||||
|
require.NoError(suite.T(), err, clues.ToCore(err))
|
||||||
|
suite.creds = m365
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *BackupUnitSuite) TestPopulateCollections() {
|
||||||
|
var (
|
||||||
|
qp = graph.QueryParams{
|
||||||
|
Category: path.ChannelMessagesCategory, // doesn't matter which one we use.
|
||||||
|
ProtectedResource: inMock.NewProvider("group_id", "user_name"),
|
||||||
|
TenantID: suite.creds.AzureTenantID,
|
||||||
|
}
|
||||||
|
statusUpdater = func(*support.ControllerOperationStatus) {}
|
||||||
|
allScope = selectors.NewGroupsBackup(nil).Channels(selectors.Any())[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
table := []struct {
|
||||||
|
name string
|
||||||
|
mock mockBackupHandler
|
||||||
|
scope selectors.GroupsScope
|
||||||
|
failFast control.FailurePolicy
|
||||||
|
expectErr require.ErrorAssertionFunc
|
||||||
|
expectColls int
|
||||||
|
expectNewColls int
|
||||||
|
expectMetadataColls int
|
||||||
|
expectDoNotMergeColls int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "happy path, one container",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one"),
|
||||||
|
messages: testdata.StubChatMessages("msg-one"),
|
||||||
|
},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.NoError,
|
||||||
|
expectColls: 1,
|
||||||
|
expectNewColls: 1,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "happy path, many containers",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one", "two"),
|
||||||
|
messages: testdata.StubChatMessages("msg-one"),
|
||||||
|
},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.NoError,
|
||||||
|
expectColls: 2,
|
||||||
|
expectNewColls: 2,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no containers pass scope",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one"),
|
||||||
|
doNotInclude: true,
|
||||||
|
},
|
||||||
|
scope: selectors.NewGroupsBackup(nil).Channels(selectors.None())[0],
|
||||||
|
expectErr: require.NoError,
|
||||||
|
expectColls: 0,
|
||||||
|
expectNewColls: 0,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no channels",
|
||||||
|
mock: mockBackupHandler{},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.NoError,
|
||||||
|
expectColls: 0,
|
||||||
|
expectNewColls: 0,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no channel messages",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one"),
|
||||||
|
},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.NoError,
|
||||||
|
expectColls: 1,
|
||||||
|
expectNewColls: 1,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "err: deleted in flight",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one"),
|
||||||
|
messagesErr: graph.ErrDeletedInFlight,
|
||||||
|
},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.Error,
|
||||||
|
expectColls: 0,
|
||||||
|
expectNewColls: 0,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "err: other error",
|
||||||
|
mock: mockBackupHandler{
|
||||||
|
channels: testdata.StubChannels("one"),
|
||||||
|
messagesErr: assert.AnError,
|
||||||
|
},
|
||||||
|
scope: allScope,
|
||||||
|
expectErr: require.Error,
|
||||||
|
expectColls: 0,
|
||||||
|
expectNewColls: 0,
|
||||||
|
expectMetadataColls: 0,
|
||||||
|
expectDoNotMergeColls: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range table {
|
||||||
|
// for _, canMakeDeltaQueries := range []bool{true, false} {
|
||||||
|
name := test.name
|
||||||
|
|
||||||
|
// if canMakeDeltaQueries {
|
||||||
|
// name += "-delta"
|
||||||
|
// } else {
|
||||||
|
// name += "-non-delta"
|
||||||
|
// }
|
||||||
|
|
||||||
|
suite.Run(name, func() {
|
||||||
|
t := suite.T()
|
||||||
|
|
||||||
|
ctx, flush := tester.NewContext(t)
|
||||||
|
defer flush()
|
||||||
|
|
||||||
|
ctrlOpts := control.Options{FailureHandling: test.failFast}
|
||||||
|
// ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries
|
||||||
|
|
||||||
|
collections, err := populateCollections(
|
||||||
|
ctx,
|
||||||
|
qp,
|
||||||
|
test.mock,
|
||||||
|
statusUpdater,
|
||||||
|
test.mock.channels,
|
||||||
|
test.scope,
|
||||||
|
ctrlOpts,
|
||||||
|
fault.New(true))
|
||||||
|
test.expectErr(t, err, clues.ToCore(err))
|
||||||
|
assert.Len(t, collections, test.expectColls, "number of collections")
|
||||||
|
|
||||||
|
// collection assertions
|
||||||
|
|
||||||
|
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
|
||||||
|
for _, c := range collections {
|
||||||
|
if c.FullPath().Service() == path.GroupsMetadataService {
|
||||||
|
metadatas++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.State() == data.DeletedState {
|
||||||
|
deleteds++
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.State() == data.NewState {
|
||||||
|
news++
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.DoNotMergeItems() {
|
||||||
|
doNotMerges++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Zero(t, deleteds, "deleted collections")
|
||||||
|
assert.Equal(t, test.expectNewColls, news, "new collections")
|
||||||
|
assert.Equal(t, test.expectMetadataColls, metadatas, "metadata collections")
|
||||||
|
assert.Equal(t, test.expectDoNotMergeColls, doNotMerges, "doNotMerge collections")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// }
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Integration tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type BackupIntgSuite struct {
|
||||||
|
tester.Suite
|
||||||
|
resource string
|
||||||
|
tenantID string
|
||||||
|
ac api.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupIntgSuite(t *testing.T) {
|
||||||
|
suite.Run(t, &BackupIntgSuite{
|
||||||
|
Suite: tester.NewIntegrationSuite(
|
||||||
|
t,
|
||||||
|
[][]string{tconfig.M365AcctCredEnvs}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *BackupIntgSuite) SetupSuite() {
|
||||||
|
t := suite.T()
|
||||||
|
|
||||||
|
ctx, flush := tester.NewContext(t)
|
||||||
|
defer flush()
|
||||||
|
|
||||||
|
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
||||||
|
|
||||||
|
suite.resource = tconfig.M365TeamID(t)
|
||||||
|
|
||||||
|
acct := tconfig.NewM365Account(t)
|
||||||
|
creds, err := acct.M365Config()
|
||||||
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
|
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
||||||
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
|
suite.tenantID = creds.AzureTenantID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *BackupIntgSuite) TestCreateCollections() {
|
||||||
|
var (
|
||||||
|
protectedResource = tconfig.M365GroupID(suite.T())
|
||||||
|
resources = []string{protectedResource}
|
||||||
|
handler = NewChannelBackupHandler(protectedResource, suite.ac.Channels())
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
scope selectors.GroupsScope
|
||||||
|
channelNames map[string]struct{}
|
||||||
|
canMakeDeltaQueries bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "channel messages non-delta",
|
||||||
|
scope: selTD.GroupsBackupChannelScope(selectors.NewGroupsBackup(resources))[0],
|
||||||
|
channelNames: map[string]struct{}{
|
||||||
|
selTD.TestChannelName: {},
|
||||||
|
},
|
||||||
|
canMakeDeltaQueries: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
suite.Run(test.name, func() {
|
||||||
|
t := suite.T()
|
||||||
|
|
||||||
|
ctx, flush := tester.NewContext(t)
|
||||||
|
defer flush()
|
||||||
|
|
||||||
|
ctrlOpts := control.DefaultOptions()
|
||||||
|
ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries
|
||||||
|
|
||||||
|
sel := selectors.NewGroupsBackup([]string{protectedResource})
|
||||||
|
sel.Include(selTD.GroupsBackupChannelScope(sel))
|
||||||
|
|
||||||
|
bpc := inject.BackupProducerConfig{
|
||||||
|
LastBackupVersion: version.NoBackup,
|
||||||
|
Options: ctrlOpts,
|
||||||
|
ProtectedResource: inMock.NewProvider(protectedResource, protectedResource),
|
||||||
|
Selector: sel.Selector,
|
||||||
|
}
|
||||||
|
|
||||||
|
collections, err := CreateCollections(
|
||||||
|
ctx,
|
||||||
|
bpc,
|
||||||
|
handler,
|
||||||
|
suite.tenantID,
|
||||||
|
test.scope,
|
||||||
|
func(status *support.ControllerOperationStatus) {},
|
||||||
|
fault.New(true))
|
||||||
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
require.NotEmpty(t, collections, "must have at least one collection")
|
||||||
|
|
||||||
|
for _, c := range collections {
|
||||||
|
if c.FullPath().Service() == path.GroupsMetadataService {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NotEmpty(t, c.FullPath().Folder(false))
|
||||||
|
|
||||||
|
fmt.Printf("\n-----\nfolder %+v\n-----\n", c.FullPath().Folder(false))
|
||||||
|
|
||||||
|
// TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection
|
||||||
|
// interface.
|
||||||
|
if !assert.Implements(t, (*data.LocationPather)(nil), c) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
loc := c.(data.LocationPather).LocationPath().String()
|
||||||
|
|
||||||
|
fmt.Printf("\n-----\nloc %+v\n-----\n", c.(data.LocationPather).LocationPath().String())
|
||||||
|
|
||||||
|
require.NotEmpty(t, loc)
|
||||||
|
|
||||||
|
delete(test.channelNames, loc)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Empty(t, test.channelNames)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
65
src/internal/m365/collection/groups/channel_handler.go
Normal file
65
src/internal/m365/collection/groups/channel_handler.go
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
package groups
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ backupHandler = &channelsBackupHandler{}
|
||||||
|
|
||||||
|
type channelsBackupHandler struct {
|
||||||
|
ac api.Channels
|
||||||
|
protectedResource string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewChannelBackupHandler(
|
||||||
|
protectedResource string,
|
||||||
|
ac api.Channels,
|
||||||
|
) channelsBackupHandler {
|
||||||
|
return channelsBackupHandler{
|
||||||
|
ac: ac,
|
||||||
|
protectedResource: protectedResource,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh channelsBackupHandler) getChannels(
|
||||||
|
ctx context.Context,
|
||||||
|
) ([]models.Channelable, error) {
|
||||||
|
return bh.ac.GetChannels(ctx, bh.protectedResource)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh channelsBackupHandler) getChannelMessagesDelta(
|
||||||
|
ctx context.Context,
|
||||||
|
channelID, prevDelta string,
|
||||||
|
) ([]models.ChatMessageable, api.DeltaUpdate, error) {
|
||||||
|
return bh.ac.GetChannelMessagesDelta(ctx, bh.protectedResource, channelID, prevDelta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh channelsBackupHandler) includeContainer(
|
||||||
|
ctx context.Context,
|
||||||
|
qp graph.QueryParams,
|
||||||
|
ch models.Channelable,
|
||||||
|
scope selectors.GroupsScope,
|
||||||
|
) bool {
|
||||||
|
return scope.Matches(selectors.GroupsChannel, ptr.Val(ch.GetDisplayName()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (bh channelsBackupHandler) canonicalPath(
|
||||||
|
folders *path.Builder,
|
||||||
|
tenantID string,
|
||||||
|
) (path.Path, error) {
|
||||||
|
return folders.
|
||||||
|
ToDataLayerPath(
|
||||||
|
tenantID,
|
||||||
|
bh.protectedResource,
|
||||||
|
path.GroupsService,
|
||||||
|
path.ChannelMessagesCategory,
|
||||||
|
false)
|
||||||
|
}
|
||||||
@ -3,38 +3,39 @@ package groups
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/microsoft/kiota-abstractions-go/serialization"
|
|
||||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BackupHandler interface {
|
type backupHandler interface {
|
||||||
GetChannelByID(
|
// gets all channels for the group
|
||||||
|
getChannels(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
teamID, channelID string,
|
) ([]models.Channelable, error)
|
||||||
) (models.Channelable, error)
|
|
||||||
NewChannelsPager(
|
|
||||||
teamID string,
|
|
||||||
) api.Pager[models.Channelable]
|
|
||||||
|
|
||||||
GetMessageByID(
|
// gets all messages by delta in the channel.
|
||||||
|
getChannelMessagesDelta(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
teamID, channelID, itemID string,
|
channelID, prevDelta string,
|
||||||
) (models.ChatMessageable, error)
|
) ([]models.ChatMessageable, api.DeltaUpdate, error)
|
||||||
NewMessagePager(
|
|
||||||
teamID, channelID string,
|
|
||||||
) api.DeltaPager[models.ChatMessageable]
|
|
||||||
|
|
||||||
GetMessageReplies(
|
// includeContainer evaluates whether the channel is included
|
||||||
|
// in the provided scope.
|
||||||
|
includeContainer(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
teamID, channelID, messageID string,
|
qp graph.QueryParams,
|
||||||
) (serialization.Parsable, error)
|
ch models.Channelable,
|
||||||
}
|
scope selectors.GroupsScope,
|
||||||
|
) bool
|
||||||
type BackupMessagesHandler interface {
|
|
||||||
GetMessage(ctx context.Context, teamID, channelID, itemID string) (models.ChatMessageable, error)
|
// canonicalPath constructs the service and category specific path for
|
||||||
NewMessagePager(teamID, channelID string) api.DeltaPager[models.ChatMessageable]
|
// the given builder.
|
||||||
GetChannel(ctx context.Context, teamID, channelID string) (models.Channelable, error)
|
canonicalPath(
|
||||||
GetReply(ctx context.Context, teamID, channelID, messageID string) (serialization.Parsable, error)
|
folders *path.Builder,
|
||||||
|
tenantID string,
|
||||||
|
) (path.Path, error)
|
||||||
}
|
}
|
||||||
|
|||||||
40
src/internal/m365/collection/groups/testdata/channels.go
vendored
Normal file
40
src/internal/m365/collection/groups/testdata/channels.go
vendored
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package testdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
|
)
|
||||||
|
|
||||||
|
func StubChannels(names ...string) []models.Channelable {
|
||||||
|
sl := make([]models.Channelable, 0, len(names))
|
||||||
|
|
||||||
|
for _, name := range names {
|
||||||
|
ch := models.NewChannel()
|
||||||
|
ch.SetDisplayName(ptr.To(name))
|
||||||
|
ch.SetId(ptr.To(uuid.NewString()))
|
||||||
|
|
||||||
|
sl = append(sl, ch)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sl
|
||||||
|
}
|
||||||
|
|
||||||
|
func StubChatMessages(names ...string) []models.ChatMessageable {
|
||||||
|
sl := make([]models.ChatMessageable, 0, len(names))
|
||||||
|
|
||||||
|
for _, name := range names {
|
||||||
|
cm := models.NewChatMessage()
|
||||||
|
cm.SetId(ptr.To(uuid.NewString()))
|
||||||
|
|
||||||
|
body := models.NewItemBody()
|
||||||
|
body.SetContent(ptr.To(name))
|
||||||
|
|
||||||
|
cm.SetBody(body)
|
||||||
|
|
||||||
|
sl = append(sl, cm)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sl
|
||||||
|
}
|
||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
"github.com/alcionai/corso/src/internal/data"
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/collection/groups"
|
||||||
"github.com/alcionai/corso/src/internal/m365/collection/site"
|
"github.com/alcionai/corso/src/internal/m365/collection/site"
|
||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
"github.com/alcionai/corso/src/internal/m365/support"
|
"github.com/alcionai/corso/src/internal/m365/support"
|
||||||
@ -94,6 +95,20 @@ func ProduceBackupCollections(
|
|||||||
el.AddRecoverable(ctx, err)
|
el.AddRecoverable(ctx, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case path.ChannelMessagesCategory:
|
||||||
|
dbcs, err = groups.CreateCollections(
|
||||||
|
ctx,
|
||||||
|
bpc,
|
||||||
|
groups.NewChannelBackupHandler(bpc.ProtectedResource.ID(), ac.Channels()),
|
||||||
|
creds.AzureTenantID,
|
||||||
|
scope,
|
||||||
|
su,
|
||||||
|
errs)
|
||||||
|
if err != nil {
|
||||||
|
el.AddRecoverable(ctx, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
collections = append(collections, dbcs...)
|
collections = append(collections, dbcs...)
|
||||||
|
|||||||
@ -222,21 +222,24 @@ func (s *groups) AllData() []GroupsScope {
|
|||||||
return scopes
|
return scopes
|
||||||
}
|
}
|
||||||
|
|
||||||
// Channel produces one or more SharePoint channel scopes, where the channel
|
// Channels produces one or more SharePoint channel scopes, where the channel
|
||||||
// matches upon a given channel by ID or Name. In order to ensure channel selection
|
// matches upon a given channel by ID or Name. In order to ensure channel selection
|
||||||
// this should always be embedded within the Filter() set; include(channel()) will
|
// this should always be embedded within the Filter() set; include(channel()) will
|
||||||
// select all items in the channel without further filtering.
|
// select all items in the channel without further filtering.
|
||||||
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
||||||
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
|
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
|
||||||
// If any slice is empty, it defaults to [selectors.None]
|
// If any slice is empty, it defaults to [selectors.None]
|
||||||
func (s *groups) Channel(channel string) []GroupsScope {
|
func (s *groups) Channels(channels []string, opts ...option) []GroupsScope {
|
||||||
return []GroupsScope{
|
var (
|
||||||
makeInfoScope[GroupsScope](
|
scopes = []GroupsScope{}
|
||||||
GroupsChannel,
|
os = append([]option{pathComparator()}, opts...)
|
||||||
GroupsInfoChannel,
|
)
|
||||||
[]string{channel},
|
|
||||||
filters.Equal),
|
scopes = append(
|
||||||
}
|
scopes,
|
||||||
|
makeScope[GroupsScope](GroupsChannel, channels, os...))
|
||||||
|
|
||||||
|
return scopes
|
||||||
}
|
}
|
||||||
|
|
||||||
// ChannelMessages produces one or more Groups channel message scopes.
|
// ChannelMessages produces one or more Groups channel message scopes.
|
||||||
|
|||||||
4
src/pkg/selectors/testdata/groups.go
vendored
4
src/pkg/selectors/testdata/groups.go
vendored
@ -4,7 +4,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
)
|
)
|
||||||
|
|
||||||
const TestChannelName = "test"
|
const TestChannelName = "Test"
|
||||||
|
|
||||||
// GroupsBackupFolderScope is the standard folder scope that should be used
|
// GroupsBackupFolderScope is the standard folder scope that should be used
|
||||||
// in integration backups with groups when interacting with libraries.
|
// in integration backups with groups when interacting with libraries.
|
||||||
@ -15,5 +15,5 @@ func GroupsBackupLibraryFolderScope(sel *selectors.GroupsBackup) []selectors.Gro
|
|||||||
// GroupsBackupChannelScope is the standard folder scope that should be used
|
// GroupsBackupChannelScope is the standard folder scope that should be used
|
||||||
// in integration backups with groups when interacting with channels.
|
// in integration backups with groups when interacting with channels.
|
||||||
func GroupsBackupChannelScope(sel *selectors.GroupsBackup) []selectors.GroupsScope {
|
func GroupsBackupChannelScope(sel *selectors.GroupsBackup) []selectors.GroupsScope {
|
||||||
return sel.Channel(TestChannelName)
|
return sel.Channels([]string{TestChannelName})
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user