Differentiate file suffixes between channel messages and conversations

This commit is contained in:
Abhishek Pandey 2024-01-26 04:44:56 -08:00
parent 7477c58698
commit 5230e60b24
6 changed files with 46 additions and 8 deletions

View File

@ -61,6 +61,11 @@ func (bh mockBackupHandler) augmentItemInfo(
// no-op
}
//lint:ignore U1000 false linter issue due to generics
func (bh mockBackupHandler) supportsItemMetadata() bool {
return false
}
func (bh mockBackupHandler) canMakeDeltaQueries() bool {
return true
}

View File

@ -125,6 +125,12 @@ func (bh channelsBackupHandler) augmentItemInfo(
// no-op
}
//lint:ignore U1000 false linter issue due to generics
func (bh channelsBackupHandler) supportsItemMetadata() bool {
// No .data and .meta files for channel messages
return false
}
func channelContainer(ch models.Channelable) container[models.Channelable] {
return container[models.Channelable]{
storageDirFolders: path.Elements{ptr.Val(ch.GetId())},

View File

@ -348,8 +348,12 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
// deleted items in a conversation. It might be added in the future
// if graph supports it, so make sure we put up both .data and .meta
// files for deletions.
col.stream <- data.NewDeletedItem(id + metadata.DataFileSuffix)
col.stream <- data.NewDeletedItem(id + metadata.MetaFileSuffix)
if col.getAndAugment.supportsItemMetadata() {
col.stream <- data.NewDeletedItem(id + metadata.DataFileSuffix)
col.stream <- data.NewDeletedItem(id + metadata.MetaFileSuffix)
} else {
col.stream <- data.NewDeletedItem(id)
}
atomic.AddInt64(&streamedItems, 1)
col.Counter.Inc(count.StreamItemsRemoved)
@ -378,6 +382,10 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
"item_id", id,
"parent_path", path.LoggableDir(col.LocationPath().String()))
// Conversation posts carry a .data suffix, while channel messages
// don't have any suffix. Metadata files are only supported for conversations.
dataFile := id
// Handle metadata before data so that if metadata file fails,
// we are not left with an orphaned data file.
//
@ -396,15 +404,14 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
if err != nil && !errors.Is(err, errMetadataFilesNotSupported) {
errs.AddRecoverable(ctx, clues.StackWC(ctx, err))
return
}
return
}
if err == nil {
// Skip adding progress reader for metadata files. It doesn't add
// much value.
storeItem, err := data.NewPrefetchedItem(
itemMeta,
id+metadata.MetaFileSuffix,
metaFile,
// Use the same last modified time as post's.
modTime)
if err != nil {
@ -422,12 +429,12 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
modTime: modTime,
getAndAugment: col.getAndAugment,
resourceID: col.protectedResource,
itemID: id,
itemID: dataFile,
containerIDs: col.FullPath().Folders(),
contains: col.contains,
parentPath: col.LocationPath().String(),
},
id+metadata.DataFileSuffix,
dataFile,
modTime,
col.Counter,
el)

View File

@ -180,6 +180,11 @@ func (getAndAugmentChannelMessage) augmentItemInfo(*details.GroupsInfo, models.C
// no-op
}
//lint:ignore U1000 false linter issue due to generics
func (getAndAugmentChannelMessage) supportsItemMetadata() bool {
return false
}
func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() {
var (
t = suite.T()
@ -322,6 +327,11 @@ func (m *getAndAugmentConversation) augmentItemInfo(*details.GroupsInfo, models.
// no-op
}
//lint:ignore U1000 false linter issue due to generics
func (m *getAndAugmentConversation) supportsItemMetadata() bool {
return true
}
func (m *getAndAugmentConversation) check(t *testing.T, expected []string) {
// Sort before comparing. We could use a set, but that would prevent us from
// detecting duplicates.

View File

@ -170,6 +170,11 @@ func (bh conversationsBackupHandler) augmentItemInfo(
dgi.Post.Topic = ptr.Val(c.GetTopic())
}
//lint:ignore U1000 false linter issue due to generics
func (bh conversationsBackupHandler) supportsItemMetadata() bool {
return true
}
func conversationThreadContainer(
c models.Conversationable,
t models.ConversationThreadable,

View File

@ -36,6 +36,7 @@ type getItemAndAugmentInfoer[C graph.GetIDer, I groupsItemer] interface {
getItemer[I]
getItemMetadataer[C, I]
augmentItemInfoer[C]
supportsItemMetadataer[C, I]
}
type augmentItemInfoer[C graph.GetIDer] interface {
@ -60,6 +61,10 @@ type getItemMetadataer[C graph.GetIDer, I groupsItemer] interface {
) (io.ReadCloser, int, error)
}
type supportsItemMetadataer[C graph.GetIDer, I groupsItemer] interface {
supportsItemMetadata() bool
}
// gets all containers for the resource
type getContainerser[C graph.GetIDer] interface {
getContainers(