diff --git a/src/internal/m365/collection/groups/backup_test.go b/src/internal/m365/collection/groups/backup_test.go index 442904fc2..b44c78037 100644 --- a/src/internal/m365/collection/groups/backup_test.go +++ b/src/internal/m365/collection/groups/backup_test.go @@ -2,6 +2,7 @@ package groups import ( "context" + "io" "testing" "time" @@ -31,6 +32,7 @@ import ( selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + groupmeta "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -136,6 +138,13 @@ func (bh mockBackupHandler) getItem( return bh.messages[itemID], bh.info[itemID], bh.getMessageErr[itemID] } +func (bh mockBackupHandler) getItemMetadata( + _ context.Context, + _ models.Channelable, +) (io.ReadCloser, int, error) { + return nil, 0, groupmeta.ErrMetadataFilesNotSupported +} + // --------------------------------------------------------------------------- // Unit Suite // --------------------------------------------------------------------------- diff --git a/src/internal/m365/collection/groups/channel_handler.go b/src/internal/m365/collection/groups/channel_handler.go index 59e97d2ee..3780d3f63 100644 --- a/src/internal/m365/collection/groups/channel_handler.go +++ b/src/internal/m365/collection/groups/channel_handler.go @@ -2,6 +2,7 @@ package groups import ( "context" + "io" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -11,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -105,6 +107,17 @@ func (bh channelsBackupHandler) getItem( return bh.ac.GetChannelMessage(ctx, groupID, containerIDs[0], messageID) } +// Channel messages don't carry metadata files. Return unsupported error. +// Adding this method for interface compliance. +// +//lint:ignore U1000 false linter issue due to generics +func (bh channelsBackupHandler) getItemMetadata( + _ context.Context, + _ models.Channelable, +) (io.ReadCloser, int, error) { + return nil, 0, metadata.ErrMetadataFilesNotSupported +} + //lint:ignore U1000 false linter issue due to generics func (bh channelsBackupHandler) augmentItemInfo( dgi *details.GroupsInfo, diff --git a/src/internal/m365/collection/groups/collection.go b/src/internal/m365/collection/groups/collection.go index fdcfe72ec..a44605fa8 100644 --- a/src/internal/m365/collection/groups/collection.go +++ b/src/internal/m365/collection/groups/collection.go @@ -23,6 +23,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" ) var ( @@ -370,6 +371,39 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau "item_id", id, "parent_path", path.LoggableDir(col.LocationPath().String())) + // Handle metadata before data so that if metadata file fails, + // we are not left with an orphaned data file. + itemMeta, _, err := col.getAndAugment.getItemMetadata( + ictx, + col.contains.container) + if err != nil && !errors.Is(err, metadata.ErrMetadataFilesNotSupported) { + errs.AddRecoverable(ctx, clues.StackWC(ctx, err)) + + return + } + + if err == nil { + // Skip adding progress reader for metadata files. It doesn't add + // much value. + storeItem, err := data.NewPrefetchedItem( + itemMeta, + id+metadata.MetaFileSuffix, + // Use the same last modified time as post's. + modTime) + if err != nil { + errs.AddRecoverable(ctx, clues.StackWC(ctx, err)) + + return + } + + col.stream <- storeItem + } + + // TODO(pandeyabs): Persist as .data file for conversations only, not for channels + // i.e. only add the .data suffix for conv backups. + // This is safe for now since channels don't utilize lazy reader yet. + id += metadata.DataFileSuffix + col.stream <- data.NewLazyItemWithInfo( ictx, &lazyItemGetter[C, I]{ diff --git a/src/internal/m365/collection/groups/collection_test.go b/src/internal/m365/collection/groups/collection_test.go index ea2ae47f6..9108c14b3 100644 --- a/src/internal/m365/collection/groups/collection_test.go +++ b/src/internal/m365/collection/groups/collection_test.go @@ -25,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/errs/core" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" ) type CollectionUnitSuite struct { @@ -165,6 +166,22 @@ func (m getAndAugmentChannelMessage) getItem( return msg, &details.GroupsInfo{}, m.Err } +//lint:ignore U1000 false linter issue due to generics +func (getAndAugmentChannelMessage) getItemMetadata( + _ context.Context, + _ models.Channelable, +) (io.ReadCloser, int, error) { + return nil, 0, metadata.ErrMetadataFilesNotSupported +} + +//lint:ignore U1000 false linter issue due to generics +func (m *getAndAugmentConversation) getItemMetadata( + _ context.Context, + _ models.Conversationable, +) (io.ReadCloser, int, error) { + return nil, 0, nil +} + //lint:ignore U1000 false linter issue due to generics func (getAndAugmentChannelMessage) augmentItemInfo(*details.GroupsInfo, models.Channelable) { // no-op diff --git a/src/internal/m365/collection/groups/conversation_handler.go b/src/internal/m365/collection/groups/conversation_handler.go index d5b948ef1..c8ada1966 100644 --- a/src/internal/m365/collection/groups/conversation_handler.go +++ b/src/internal/m365/collection/groups/conversation_handler.go @@ -1,7 +1,10 @@ package groups import ( + "bytes" "context" + "encoding/json" + "io" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -11,6 +14,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -132,6 +136,24 @@ func (bh conversationsBackupHandler) getItem( postID) } +//lint:ignore U1000 false linter issue due to generics +func (bh conversationsBackupHandler) getItemMetadata( + ctx context.Context, + c models.Conversationable, +) (io.ReadCloser, int, error) { + meta := metadata.ConversationPostMetadata{ + Recipients: []string{bh.resourceEmail}, + Topic: ptr.Val(c.GetTopic()), + } + + metaJSON, err := json.Marshal(meta) + if err != nil { + return nil, 0, clues.WrapWC(ctx, err, "serializing post metadata") + } + + return io.NopCloser(bytes.NewReader(metaJSON)), len(metaJSON), nil +} + //lint:ignore U1000 false linter issue due to generics func (bh conversationsBackupHandler) augmentItemInfo( dgi *details.GroupsInfo, diff --git a/src/internal/m365/collection/groups/handlers.go b/src/internal/m365/collection/groups/handlers.go index dfb224d56..21463f77b 100644 --- a/src/internal/m365/collection/groups/handlers.go +++ b/src/internal/m365/collection/groups/handlers.go @@ -2,6 +2,7 @@ package groups import ( "context" + "io" "github.com/microsoft/kiota-abstractions-go/serialization" @@ -33,6 +34,7 @@ type backupHandler[C graph.GetIDer, I groupsItemer] interface { type getItemAndAugmentInfoer[C graph.GetIDer, I groupsItemer] interface { getItemer[I] + getItemMetadataer[C, I] augmentItemInfoer[C] } @@ -51,6 +53,13 @@ type getItemer[I groupsItemer] interface { ) (I, *details.GroupsInfo, error) } +type getItemMetadataer[C graph.GetIDer, I groupsItemer] interface { + getItemMetadata( + ctx context.Context, + c C, + ) (io.ReadCloser, int, error) +} + // gets all containers for the resource type getContainerser[C graph.GetIDer] interface { getContainers( diff --git a/src/pkg/services/m365/api/graph/metadata/consts.go b/src/pkg/services/m365/api/graph/metadata/consts.go index 9ab8a6dcb..b49771689 100644 --- a/src/pkg/services/m365/api/graph/metadata/consts.go +++ b/src/pkg/services/m365/api/graph/metadata/consts.go @@ -1,6 +1,8 @@ package metadata -import "strings" +import ( + "strings" +) const ( MetaFileSuffix = ".meta" diff --git a/src/pkg/services/m365/api/graph/metadata/groups/conversations.go b/src/pkg/services/m365/api/graph/metadata/groups/conversations.go new file mode 100644 index 000000000..c8eff8122 --- /dev/null +++ b/src/pkg/services/m365/api/graph/metadata/groups/conversations.go @@ -0,0 +1,8 @@ +package metadata + +// ConversationPostMetadata stores metadata for a given conversation post, +// stored as a .meta file in kopia. +type ConversationPostMetadata struct { + Recipients []string `json:"recipients,omitempty"` + Topic string `json:"topic,omitempty"` +} diff --git a/src/pkg/services/m365/api/graph/metadata/metadata.go b/src/pkg/services/m365/api/graph/metadata/metadata.go index 843e1203b..a7772bb39 100644 --- a/src/pkg/services/m365/api/graph/metadata/metadata.go +++ b/src/pkg/services/m365/api/graph/metadata/metadata.go @@ -1,18 +1,29 @@ package metadata import ( + "github.com/alcionai/clues" + "github.com/alcionai/corso/src/pkg/path" ) +var ErrMetadataFilesNotSupported = clues.New("metadata files not supported") + func IsMetadataFile(p path.Path) bool { switch p.Service() { case path.OneDriveService: return HasMetaSuffix(p.Item()) - case path.SharePointService, path.GroupsService: + case path.SharePointService: return p.Category() == path.LibrariesCategory && HasMetaSuffix(p.Item()) + case path.GroupsService: + return p.Category() == path.LibrariesCategory && HasMetaSuffix(p.Item()) || + p.Category() == path.ConversationPostsCategory && HasMetaSuffix(p.Item()) default: return false } } + +// func WithDataSuffix(p path.Path) path.Path { +// return p.WithItem(p.Item() + ".data") +// } diff --git a/src/pkg/services/m365/api/graph/metadata/metadata_test.go b/src/pkg/services/m365/api/graph/metadata/metadata_test.go index 775a6d446..379e651d9 100644 --- a/src/pkg/services/m365/api/graph/metadata/metadata_test.go +++ b/src/pkg/services/m365/api/graph/metadata/metadata_test.go @@ -152,3 +152,108 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() { } } } + +func (suite *MetadataUnitSuite) TestIsMetadataFile() { + table := []struct { + name string + service path.ServiceType + category path.CategoryType + isMetaFile bool + expected bool + }{ + { + name: "onedrive .data file", + service: path.OneDriveService, + category: path.FilesCategory, + }, + { + name: "sharepoint library .data file", + service: path.SharePointService, + category: path.LibrariesCategory, + }, + { + name: "group library .data file", + service: path.GroupsService, + category: path.LibrariesCategory, + }, + { + name: "group conversations .data file", + service: path.GroupsService, + category: path.ConversationPostsCategory, + }, + { + name: "onedrive .meta file", + service: path.OneDriveService, + category: path.FilesCategory, + isMetaFile: true, + expected: true, + }, + { + name: "sharepoint library .meta file", + service: path.SharePointService, + category: path.LibrariesCategory, + isMetaFile: true, + expected: true, + }, + { + name: "group library .meta file", + service: path.GroupsService, + category: path.LibrariesCategory, + isMetaFile: true, + expected: true, + }, + { + name: "group conversations .meta file", + service: path.GroupsService, + category: path.ConversationPostsCategory, + isMetaFile: true, + expected: true, + }, + // For services which don't have metadata files, make sure the function + // returns false. We don't want .meta suffix (assuming it exists) in + // these cases to be interpreted as metadata files. + { + name: "exchange service", + service: path.ExchangeService, + category: path.EmailCategory, + isMetaFile: true, + }, + { + name: "group channels", + service: path.GroupsService, + category: path.ChannelMessagesCategory, + isMetaFile: true, + }, + { + name: "lists", + service: path.SharePointService, + category: path.ListsCategory, + isMetaFile: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + fileName := "file" + if test.isMetaFile { + fileName += metadata.MetaFileSuffix + } else { + fileName += metadata.DataFileSuffix + } + + p, err := path.Build( + "t", + "u", + test.service, + test.category, + true, + "some", "path", "for", fileName) + require.NoError(t, err, clues.ToCore(err)) + + actual := metadata.IsMetadataFile(p) + assert.Equal(t, test.expected, actual) + }) + } +}