larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Persist metadata files for group mailboxes

Browse Source
This commit is contained in:
Abhishek Pandey 2024-01-26 01:17:23 -08:00
parent 8133da3087
commit 41eb63686d
10 changed files with 232 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/internal/m365/collection/groups/backup_test.go
View File

@ -2,6 +2,7 @@ package groups
import (
"context"
"io"
"testing"
"time"
@ -31,6 +32,7 @@ import (
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
groupmeta "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
)
@ -136,6 +138,13 @@ func (bh mockBackupHandler) getItem(
return bh.messages[itemID], bh.info[itemID], bh.getMessageErr[itemID]
}
func (bh mockBackupHandler) getItemMetadata(
_ context.Context,
_ models.Channelable,
) (io.ReadCloser, int, error) {
return nil, 0, groupmeta.ErrMetadataFilesNotSupported
}
// ---------------------------------------------------------------------------
// Unit Suite
// ---------------------------------------------------------------------------

13
src/internal/m365/collection/groups/channel_handler.go
View File

@ -2,6 +2,7 @@ package groups
import (
"context"
"io"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -11,6 +12,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
)
@ -105,6 +107,17 @@ func (bh channelsBackupHandler) getItem(
return bh.ac.GetChannelMessage(ctx, groupID, containerIDs[0], messageID)
}
// Channel messages don't carry metadata files. Return unsupported error.
// Adding this method for interface compliance.
//
//lint:ignore U1000 false linter issue due to generics
func (bh channelsBackupHandler) getItemMetadata(
_ context.Context,
_ models.Channelable,
) (io.ReadCloser, int, error) {
return nil, 0, metadata.ErrMetadataFilesNotSupported
}
//lint:ignore U1000 false linter issue due to generics
func (bh channelsBackupHandler) augmentItemInfo(
dgi *details.GroupsInfo,

34
src/internal/m365/collection/groups/collection.go
View File

@ -23,6 +23,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
)
var (
@ -370,6 +371,39 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
"item_id", id,
"parent_path", path.LoggableDir(col.LocationPath().String()))
// Handle metadata before data so that if metadata file fails,
// we are not left with an orphaned data file.
itemMeta, _, err := col.getAndAugment.getItemMetadata(
ictx,
col.contains.container)
if err != nil && !errors.Is(err, metadata.ErrMetadataFilesNotSupported) {
errs.AddRecoverable(ctx, clues.StackWC(ctx, err))
return
}
if err == nil {
// Skip adding progress reader for metadata files. It doesn't add
// much value.
storeItem, err := data.NewPrefetchedItem(
itemMeta,
id+metadata.MetaFileSuffix,
// Use the same last modified time as post's.
modTime)
if err != nil {
errs.AddRecoverable(ctx, clues.StackWC(ctx, err))
return
}
col.stream <- storeItem
}
// TODO(pandeyabs): Persist as .data file for conversations only, not for channels
// i.e. only add the .data suffix for conv backups.
// This is safe for now since channels don't utilize lazy reader yet.
id += metadata.DataFileSuffix
col.stream <- data.NewLazyItemWithInfo(
ictx,
&lazyItemGetter[C, I]{

17
src/internal/m365/collection/groups/collection_test.go
View File

@ -25,6 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/errs/core"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
)
type CollectionUnitSuite struct {
@ -165,6 +166,22 @@ func (m getAndAugmentChannelMessage) getItem(
return msg, &details.GroupsInfo{}, m.Err
}
//lint:ignore U1000 false linter issue due to generics
func (getAndAugmentChannelMessage) getItemMetadata(
_ context.Context,
_ models.Channelable,
) (io.ReadCloser, int, error) {
return nil, 0, metadata.ErrMetadataFilesNotSupported
}
//lint:ignore U1000 false linter issue due to generics
func (m *getAndAugmentConversation) getItemMetadata(
_ context.Context,
_ models.Conversationable,
) (io.ReadCloser, int, error) {
return nil, 0, nil
}
//lint:ignore U1000 false linter issue due to generics
func (getAndAugmentChannelMessage) augmentItemInfo(*details.GroupsInfo, models.Channelable) {
// no-op

22
src/internal/m365/collection/groups/conversation_handler.go
View File

@ -1,7 +1,10 @@
package groups
import (
"bytes"
"context"
"encoding/json"
"io"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -11,6 +14,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
)
@ -132,6 +136,24 @@ func (bh conversationsBackupHandler) getItem(
postID)
}
//lint:ignore U1000 false linter issue due to generics
func (bh conversationsBackupHandler) getItemMetadata(
ctx context.Context,
c models.Conversationable,
) (io.ReadCloser, int, error) {
meta := metadata.ConversationPostMetadata{
Recipients: []string{bh.resourceEmail},
Topic: ptr.Val(c.GetTopic()),
}
metaJSON, err := json.Marshal(meta)
if err != nil {
return nil, 0, clues.WrapWC(ctx, err, "serializing post metadata")
}
return io.NopCloser(bytes.NewReader(metaJSON)), len(metaJSON), nil
}
//lint:ignore U1000 false linter issue due to generics
func (bh conversationsBackupHandler) augmentItemInfo(
dgi *details.GroupsInfo,

9
src/internal/m365/collection/groups/handlers.go
View File

@ -2,6 +2,7 @@ package groups
import (
"context"
"io"
"github.com/microsoft/kiota-abstractions-go/serialization"
@ -33,6 +34,7 @@ type backupHandler[C graph.GetIDer, I groupsItemer] interface {
type getItemAndAugmentInfoer[C graph.GetIDer, I groupsItemer] interface {
getItemer[I]
getItemMetadataer[C, I]
augmentItemInfoer[C]
}
@ -51,6 +53,13 @@ type getItemer[I groupsItemer] interface {
) (I, *details.GroupsInfo, error)
}
type getItemMetadataer[C graph.GetIDer, I groupsItemer] interface {
getItemMetadata(
ctx context.Context,
c C,
) (io.ReadCloser, int, error)
}
// gets all containers for the resource
type getContainerser[C graph.GetIDer] interface {
getContainers(

4
src/pkg/services/m365/api/graph/metadata/consts.go
View File

@ -1,6 +1,8 @@
package metadata
import "strings"
import (
"strings"
)
const (
MetaFileSuffix = ".meta"

8
src/pkg/services/m365/api/graph/metadata/groups/conversations.go Normal file
View File

@ -0,0 +1,8 @@
package metadata
// ConversationPostMetadata stores metadata for a given conversation post,
// stored as a .meta file in kopia.
type ConversationPostMetadata struct {
Recipients []string `json:"recipients,omitempty"`
Topic string `json:"topic,omitempty"`
}

13
src/pkg/services/m365/api/graph/metadata/metadata.go
View File

@ -1,18 +1,29 @@
package metadata
import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/path"
)
var ErrMetadataFilesNotSupported = clues.New("metadata files not supported")
func IsMetadataFile(p path.Path) bool {
switch p.Service() {
case path.OneDriveService:
return HasMetaSuffix(p.Item())
case path.SharePointService, path.GroupsService:
case path.SharePointService:
return p.Category() == path.LibrariesCategory && HasMetaSuffix(p.Item())
case path.GroupsService:
return p.Category() == path.LibrariesCategory && HasMetaSuffix(p.Item()) ||
p.Category() == path.ConversationPostsCategory && HasMetaSuffix(p.Item())
default:
return false
}
}
// func WithDataSuffix(p path.Path) path.Path {
// return p.WithItem(p.Item() + ".data")
// }

105
src/pkg/services/m365/api/graph/metadata/metadata_test.go
View File

@ -152,3 +152,108 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() {
}
}
}
func (suite *MetadataUnitSuite) TestIsMetadataFile() {
table := []struct {
name string
service path.ServiceType
category path.CategoryType
isMetaFile bool
expected bool
}{
{
name: "onedrive .data file",
service: path.OneDriveService,
category: path.FilesCategory,
},
{
name: "sharepoint library .data file",
service: path.SharePointService,
category: path.LibrariesCategory,
},
{
name: "group library .data file",
service: path.GroupsService,
category: path.LibrariesCategory,
},
{
name: "group conversations .data file",
service: path.GroupsService,
category: path.ConversationPostsCategory,
},
{
name: "onedrive .meta file",
service: path.OneDriveService,
category: path.FilesCategory,
isMetaFile: true,
expected: true,
},
{
name: "sharepoint library .meta file",
service: path.SharePointService,
category: path.LibrariesCategory,
isMetaFile: true,
expected: true,
},
{
name: "group library .meta file",
service: path.GroupsService,
category: path.LibrariesCategory,
isMetaFile: true,
expected: true,
},
{
name: "group conversations .meta file",
service: path.GroupsService,
category: path.ConversationPostsCategory,
isMetaFile: true,
expected: true,
},
// For services which don't have metadata files, make sure the function
// returns false. We don't want .meta suffix (assuming it exists) in
// these cases to be interpreted as metadata files.
{
name: "exchange service",
service: path.ExchangeService,
category: path.EmailCategory,
isMetaFile: true,
},
{
name: "group channels",
service: path.GroupsService,
category: path.ChannelMessagesCategory,
isMetaFile: true,
},
{
name: "lists",
service: path.SharePointService,
category: path.ListsCategory,
isMetaFile: true,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
fileName := "file"
if test.isMetaFile {
fileName += metadata.MetaFileSuffix
} else {
fileName += metadata.DataFileSuffix
}
p, err := path.Build(
"t",
"u",
test.service,
test.category,
true,
"some", "path", "for", fileName)
require.NoError(t, err, clues.ToCore(err))
actual := metadata.IsMetadataFile(p)
assert.Equal(t, test.expected, actual)
})
}
}