From 3e2268e5241e0599d032551dc0ba851799832cc3 Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Fri, 26 Jan 2024 14:18:11 -0800 Subject: [PATCH] Move conv metadata definiton --- src/internal/m365/collection/groups/backup_test.go | 3 +-- .../m365/collection/groups/channel_handler.go | 3 +-- src/internal/m365/collection/groups/collection.go | 14 +++++++++++++- .../m365/collection/groups/collection_test.go | 7 +++---- .../m365/collection/groups/conversation_handler.go | 2 +- .../collection/groups/conversation_handler_test.go | 2 +- .../collection/groups/metadata}/conversations.go | 0 .../services/m365/api/graph/metadata/metadata.go | 4 ---- 8 files changed, 20 insertions(+), 15 deletions(-) rename src/{pkg/services/m365/api/graph/metadata/groups => internal/m365/collection/groups/metadata}/conversations.go (100%) diff --git a/src/internal/m365/collection/groups/backup_test.go b/src/internal/m365/collection/groups/backup_test.go index b44c78037..c3cf1890e 100644 --- a/src/internal/m365/collection/groups/backup_test.go +++ b/src/internal/m365/collection/groups/backup_test.go @@ -32,7 +32,6 @@ import ( selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" - groupmeta "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -142,7 +141,7 @@ func (bh mockBackupHandler) getItemMetadata( _ context.Context, _ models.Channelable, ) (io.ReadCloser, int, error) { - return nil, 0, groupmeta.ErrMetadataFilesNotSupported + return nil, 0, errMetadataFilesNotSupported } // --------------------------------------------------------------------------- diff --git a/src/internal/m365/collection/groups/channel_handler.go b/src/internal/m365/collection/groups/channel_handler.go index 3780d3f63..4b0adc16a 100644 --- a/src/internal/m365/collection/groups/channel_handler.go +++ b/src/internal/m365/collection/groups/channel_handler.go @@ -12,7 +12,6 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" - "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -115,7 +114,7 @@ func (bh channelsBackupHandler) getItemMetadata( _ context.Context, _ models.Channelable, ) (io.ReadCloser, int, error) { - return nil, 0, metadata.ErrMetadataFilesNotSupported + return nil, 0, errMetadataFilesNotSupported } //lint:ignore U1000 false linter issue due to generics diff --git a/src/internal/m365/collection/groups/collection.go b/src/internal/m365/collection/groups/collection.go index 092ae5bd0..25d61f6fb 100644 --- a/src/internal/m365/collection/groups/collection.go +++ b/src/internal/m365/collection/groups/collection.go @@ -31,6 +31,8 @@ var ( _ data.BackupCollection = &lazyFetchCollection[graph.GetIDer, groupsItemer]{} ) +var errMetadataFilesNotSupported = clues.New("metadata files not supported") + const ( collectionChannelBufferSize = 1000 numberOfRetries = 4 @@ -378,10 +380,20 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau // Handle metadata before data so that if metadata file fails, // we are not left with an orphaned data file. + // + // If the data download fails for some reason other than deleted in + // flight, we will still end up persisting a .meta file. This is + // fine however, since the next backup will overwrite it. + // + // If item is deleted in flight, we will end up with an orphaned + // .meta file. The only impact here is storage bloat, which + // is minimal. Other impact could be if we do an in-order restore + // using a tree built from .meta files. We may have some .meta + // files without corresponding .data files. itemMeta, _, err := col.getAndAugment.getItemMetadata( ictx, col.contains.container) - if err != nil && !errors.Is(err, metadata.ErrMetadataFilesNotSupported) { + if err != nil && !errors.Is(err, errMetadataFilesNotSupported) { errs.AddRecoverable(ctx, clues.StackWC(ctx, err)) return diff --git a/src/internal/m365/collection/groups/collection_test.go b/src/internal/m365/collection/groups/collection_test.go index af45ad865..4a4803ac9 100644 --- a/src/internal/m365/collection/groups/collection_test.go +++ b/src/internal/m365/collection/groups/collection_test.go @@ -27,7 +27,6 @@ import ( "github.com/alcionai/corso/src/pkg/errs/core" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" ) type CollectionUnitSuite struct { @@ -173,7 +172,7 @@ func (getAndAugmentChannelMessage) getItemMetadata( _ context.Context, _ models.Channelable, ) (io.ReadCloser, int, error) { - return nil, 0, metadata.ErrMetadataFilesNotSupported + return nil, 0, errMetadataFilesNotSupported } //lint:ignore U1000 false linter issue due to generics @@ -290,6 +289,7 @@ func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() { type getAndAugmentConversation struct { GetItemErr error + GetMetaErr error CallIDs []string } @@ -313,8 +313,7 @@ func (m *getAndAugmentConversation) getItemMetadata( _ context.Context, _ models.Conversationable, ) (io.ReadCloser, int, error) { - // Return some dummy data - return io.NopCloser(strings.NewReader("test")), 4, nil + return io.NopCloser(strings.NewReader("test")), 4, m.GetMetaErr } // diff --git a/src/internal/m365/collection/groups/conversation_handler.go b/src/internal/m365/collection/groups/conversation_handler.go index c8ada1966..edba3fa8a 100644 --- a/src/internal/m365/collection/groups/conversation_handler.go +++ b/src/internal/m365/collection/groups/conversation_handler.go @@ -10,11 +10,11 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/collection/groups/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" - metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) diff --git a/src/internal/m365/collection/groups/conversation_handler_test.go b/src/internal/m365/collection/groups/conversation_handler_test.go index cd70b7cb9..d33ceea82 100644 --- a/src/internal/m365/collection/groups/conversation_handler_test.go +++ b/src/internal/m365/collection/groups/conversation_handler_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/collection/groups/metadata" "github.com/alcionai/corso/src/internal/tester" - metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups" ) type ConversationHandlerUnitSuite struct { diff --git a/src/pkg/services/m365/api/graph/metadata/groups/conversations.go b/src/internal/m365/collection/groups/metadata/conversations.go similarity index 100% rename from src/pkg/services/m365/api/graph/metadata/groups/conversations.go rename to src/internal/m365/collection/groups/metadata/conversations.go diff --git a/src/pkg/services/m365/api/graph/metadata/metadata.go b/src/pkg/services/m365/api/graph/metadata/metadata.go index 4d3dd8a0b..4fa23e087 100644 --- a/src/pkg/services/m365/api/graph/metadata/metadata.go +++ b/src/pkg/services/m365/api/graph/metadata/metadata.go @@ -1,13 +1,9 @@ package metadata import ( - "github.com/alcionai/clues" - "github.com/alcionai/corso/src/pkg/path" ) -var ErrMetadataFilesNotSupported = clues.New("metadata files not supported") - func IsMetadataFile(p path.Path) bool { switch p.Service() { case path.OneDriveService: