With conv export
This commit is contained in:
Abhishek Pandey
parent
c8adfc8f72
commit
c4984e9309
@ -3,13 +3,11 @@ package export
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
|
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -23,167 +21,167 @@ func CheckGroupsExport(
|
|||||||
// assumes we only need to sanity check the default site.
|
// assumes we only need to sanity check the default site.
|
||||||
// should we expand this to check all sites in the group?
|
// should we expand this to check all sites in the group?
|
||||||
// are we backing up / restoring more than the default site?
|
// are we backing up / restoring more than the default site?
|
||||||
site, err := ac.Sites().GetByID(ctx, envs.TeamSiteID, api.CallConfig{})
|
// site, err := ac.Sites().GetByID(ctx, envs.TeamSiteID, api.CallConfig{})
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
common.Fatal(ctx, "getting the drive:", err)
|
// common.Fatal(ctx, "getting the drive:", err)
|
||||||
}
|
// }
|
||||||
|
|
||||||
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.TeamSiteID)
|
// drive, err := ac.Sites().GetDefaultDrive(ctx, envs.TeamSiteID)
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
common.Fatal(ctx, "getting the drive:", err)
|
// common.Fatal(ctx, "getting the drive:", err)
|
||||||
}
|
// }
|
||||||
|
|
||||||
// checkChannelMessagesExport(
|
// checkChannelMessagesExport(
|
||||||
// ctx,
|
// ctx,
|
||||||
// ac,
|
// ac,
|
||||||
// envs)
|
// envs)
|
||||||
|
|
||||||
// checkConversationPostExports(
|
checkConversationPostExports(
|
||||||
|
ctx,
|
||||||
|
ac,
|
||||||
|
envs)
|
||||||
|
|
||||||
|
// envs.RestoreContainer = filepath.Join(
|
||||||
|
// envs.RestoreContainer,
|
||||||
|
// "Libraries",
|
||||||
|
// ptr.Val(site.GetName()),
|
||||||
|
// "Documents") // check in default loc
|
||||||
|
// driveish.CheckExport(
|
||||||
// ctx,
|
// ctx,
|
||||||
// ac,
|
// ac,
|
||||||
|
// drive,
|
||||||
// envs)
|
// envs)
|
||||||
|
|
||||||
envs.RestoreContainer = filepath.Join(
|
|
||||||
envs.RestoreContainer,
|
|
||||||
"Libraries",
|
|
||||||
ptr.Val(site.GetName()),
|
|
||||||
"Documents") // check in default loc
|
|
||||||
driveish.CheckExport(
|
|
||||||
ctx,
|
|
||||||
ac,
|
|
||||||
drive,
|
|
||||||
envs)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func removeDeletedMessages(
|
// func removeDeletedMessages(
|
||||||
ctx context.Context,
|
// ctx context.Context,
|
||||||
tree *common.Sanitree[models.Channelable, models.ChatMessageable],
|
// tree *common.Sanitree[models.Channelable, models.ChatMessageable],
|
||||||
) {
|
// ) {
|
||||||
for _, child := range tree.Children {
|
// for _, child := range tree.Children {
|
||||||
for _, leaf := range child.Leaves {
|
// for _, leaf := range child.Leaves {
|
||||||
if leaf.Self.GetDeletedDateTime() != nil {
|
// if leaf.Self.GetDeletedDateTime() != nil {
|
||||||
delete(child.Leaves, leaf.ID)
|
// delete(child.Leaves, leaf.ID)
|
||||||
child.CountLeaves--
|
// child.CountLeaves--
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
func checkChannelMessagesExport(
|
// func checkChannelMessagesExport(
|
||||||
ctx context.Context,
|
// ctx context.Context,
|
||||||
ac api.Client,
|
// ac api.Client,
|
||||||
envs common.Envs,
|
// envs common.Envs,
|
||||||
) {
|
// ) {
|
||||||
sourceTree := populateMessagesSanitree(
|
// sourceTree := populateMessagesSanitree(
|
||||||
ctx,
|
// ctx,
|
||||||
ac,
|
// ac,
|
||||||
envs.GroupID)
|
// envs.GroupID)
|
||||||
|
|
||||||
fpTree := common.BuildFilepathSanitree(ctx, envs.RestoreContainer)
|
// fpTree := common.BuildFilepathSanitree(ctx, envs.RestoreContainer)
|
||||||
|
|
||||||
comparator := func(
|
// comparator := func(
|
||||||
ctx context.Context,
|
// ctx context.Context,
|
||||||
expect *common.Sanitree[models.Channelable, models.ChatMessageable],
|
// expect *common.Sanitree[models.Channelable, models.ChatMessageable],
|
||||||
result *common.Sanitree[fs.FileInfo, fs.FileInfo],
|
// result *common.Sanitree[fs.FileInfo, fs.FileInfo],
|
||||||
) {
|
// ) {
|
||||||
for key := range expect.Leaves {
|
// for key := range expect.Leaves {
|
||||||
expect.Leaves[key].Size = 0 // msg sizes cannot be compared
|
// expect.Leaves[key].Size = 0 // msg sizes cannot be compared
|
||||||
}
|
// }
|
||||||
|
|
||||||
updatedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
|
// updatedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
|
||||||
|
|
||||||
for key, leaf := range result.Leaves {
|
// for key, leaf := range result.Leaves {
|
||||||
key = strings.TrimSuffix(key, ".json")
|
// key = strings.TrimSuffix(key, ".json")
|
||||||
leaf.Size = 0 // we cannot compare sizes
|
// leaf.Size = 0 // we cannot compare sizes
|
||||||
updatedResultLeaves[key] = leaf
|
// updatedResultLeaves[key] = leaf
|
||||||
}
|
// }
|
||||||
|
|
||||||
common.CompareLeaves(ctx, expect.Leaves, updatedResultLeaves, nil)
|
// common.CompareLeaves(ctx, expect.Leaves, updatedResultLeaves, nil)
|
||||||
}
|
// }
|
||||||
|
|
||||||
removeDeletedMessages(ctx, sourceTree)
|
// removeDeletedMessages(ctx, sourceTree)
|
||||||
|
|
||||||
common.CompareDiffTrees(
|
// common.CompareDiffTrees(
|
||||||
ctx,
|
// ctx,
|
||||||
sourceTree,
|
// sourceTree,
|
||||||
fpTree.Children["Messages"],
|
// fpTree.Children["Messages"],
|
||||||
comparator)
|
// comparator)
|
||||||
|
|
||||||
common.Infof(ctx, "Success")
|
// common.Infof(ctx, "Success")
|
||||||
}
|
// }
|
||||||
|
|
||||||
func populateMessagesSanitree(
|
// func populateMessagesSanitree(
|
||||||
ctx context.Context,
|
// ctx context.Context,
|
||||||
ac api.Client,
|
// ac api.Client,
|
||||||
groupID string,
|
// groupID string,
|
||||||
) *common.Sanitree[models.Channelable, models.ChatMessageable] {
|
// ) *common.Sanitree[models.Channelable, models.ChatMessageable] {
|
||||||
root := &common.Sanitree[models.Channelable, models.ChatMessageable]{
|
// root := &common.Sanitree[models.Channelable, models.ChatMessageable]{
|
||||||
ID: groupID,
|
// ID: groupID,
|
||||||
Name: path.ChannelMessagesCategory.HumanString(),
|
// Name: path.ChannelMessagesCategory.HumanString(),
|
||||||
// group should not have leaves
|
// // group should not have leaves
|
||||||
Children: map[string]*common.Sanitree[models.Channelable, models.ChatMessageable]{},
|
// Children: map[string]*common.Sanitree[models.Channelable, models.ChatMessageable]{},
|
||||||
}
|
// }
|
||||||
|
|
||||||
channels, err := ac.Channels().GetChannels(ctx, groupID)
|
// channels, err := ac.Channels().GetChannels(ctx, groupID)
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
common.Fatal(ctx, "getting channels", err)
|
// common.Fatal(ctx, "getting channels", err)
|
||||||
}
|
// }
|
||||||
|
|
||||||
for _, ch := range channels {
|
// for _, ch := range channels {
|
||||||
child := &common.Sanitree[
|
// child := &common.Sanitree[
|
||||||
models.Channelable, models.ChatMessageable,
|
// models.Channelable, models.ChatMessageable,
|
||||||
]{
|
// ]{
|
||||||
Parent: root,
|
// Parent: root,
|
||||||
ID: ptr.Val(ch.GetId()),
|
// ID: ptr.Val(ch.GetId()),
|
||||||
Name: ptr.Val(ch.GetDisplayName()),
|
// Name: ptr.Val(ch.GetDisplayName()),
|
||||||
Leaves: map[string]*common.Sanileaf[models.Channelable, models.ChatMessageable]{},
|
// Leaves: map[string]*common.Sanileaf[models.Channelable, models.ChatMessageable]{},
|
||||||
// no children in channels
|
// // no children in channels
|
||||||
}
|
// }
|
||||||
|
|
||||||
msgs, err := ac.Channels().GetChannelMessages(
|
// msgs, err := ac.Channels().GetChannelMessages(
|
||||||
ctx,
|
// ctx,
|
||||||
groupID,
|
// groupID,
|
||||||
ptr.Val(ch.GetId()),
|
// ptr.Val(ch.GetId()),
|
||||||
api.CallConfig{
|
// api.CallConfig{
|
||||||
// include all nessage replies in each message
|
// // include all nessage replies in each message
|
||||||
Expand: []string{"replies"},
|
// Expand: []string{"replies"},
|
||||||
})
|
// })
|
||||||
if err != nil {
|
// if err != nil {
|
||||||
common.Fatal(ctx, "getting channel messages", err)
|
// common.Fatal(ctx, "getting channel messages", err)
|
||||||
}
|
// }
|
||||||
|
|
||||||
filteredMsgs := []models.ChatMessageable{}
|
// filteredMsgs := []models.ChatMessageable{}
|
||||||
|
|
||||||
for _, msg := range msgs {
|
// for _, msg := range msgs {
|
||||||
// filter out system messages (we don't really work with them)
|
// // filter out system messages (we don't really work with them)
|
||||||
if api.IsNotSystemMessage(msg) {
|
// if api.IsNotSystemMessage(msg) {
|
||||||
filteredMsgs = append(filteredMsgs, msg)
|
// filteredMsgs = append(filteredMsgs, msg)
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
if len(filteredMsgs) == 0 {
|
// if len(filteredMsgs) == 0 {
|
||||||
common.Infof(ctx, "skipped empty channel: %s", ptr.Val(ch.GetDisplayName()))
|
// common.Infof(ctx, "skipped empty channel: %s", ptr.Val(ch.GetDisplayName()))
|
||||||
continue
|
// continue
|
||||||
}
|
// }
|
||||||
|
|
||||||
for _, msg := range filteredMsgs {
|
// for _, msg := range filteredMsgs {
|
||||||
child.CountLeaves++
|
// child.CountLeaves++
|
||||||
child.Leaves[ptr.Val(msg.GetId())] = &common.Sanileaf[
|
// child.Leaves[ptr.Val(msg.GetId())] = &common.Sanileaf[
|
||||||
models.Channelable,
|
// models.Channelable,
|
||||||
models.ChatMessageable,
|
// models.ChatMessageable,
|
||||||
]{
|
// ]{
|
||||||
Self: msg,
|
// Self: msg,
|
||||||
ID: ptr.Val(msg.GetId()),
|
// ID: ptr.Val(msg.GetId()),
|
||||||
Name: ptr.Val(msg.GetId()), // channel messages have no display name
|
// Name: ptr.Val(msg.GetId()), // channel messages have no display name
|
||||||
Size: int64(len(msg.GetReplies())), // size is the count of replies
|
// Size: int64(len(msg.GetReplies())), // size is the count of replies
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
root.Children[ptr.Val(ch.GetDisplayName())] = child
|
// root.Children[ptr.Val(ch.GetDisplayName())] = child
|
||||||
}
|
// }
|
||||||
|
|
||||||
return root
|
// return root
|
||||||
}
|
// }
|
||||||
|
|
||||||
func checkConversationPostExports(
|
func checkConversationPostExports(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user