Move conv metadata definiton
This commit is contained in:
parent
abca65daa9
commit
3e2268e524
@ -32,7 +32,6 @@ import (
|
|||||||
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||||
groupmeta "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
|
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -142,7 +141,7 @@ func (bh mockBackupHandler) getItemMetadata(
|
|||||||
_ context.Context,
|
_ context.Context,
|
||||||
_ models.Channelable,
|
_ models.Channelable,
|
||||||
) (io.ReadCloser, int, error) {
|
) (io.ReadCloser, int, error) {
|
||||||
return nil, 0, groupmeta.ErrMetadataFilesNotSupported
|
return nil, 0, errMetadataFilesNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@ -12,7 +12,6 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
|
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -115,7 +114,7 @@ func (bh channelsBackupHandler) getItemMetadata(
|
|||||||
_ context.Context,
|
_ context.Context,
|
||||||
_ models.Channelable,
|
_ models.Channelable,
|
||||||
) (io.ReadCloser, int, error) {
|
) (io.ReadCloser, int, error) {
|
||||||
return nil, 0, metadata.ErrMetadataFilesNotSupported
|
return nil, 0, errMetadataFilesNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
//lint:ignore U1000 false linter issue due to generics
|
//lint:ignore U1000 false linter issue due to generics
|
||||||
|
|||||||
@ -31,6 +31,8 @@ var (
|
|||||||
_ data.BackupCollection = &lazyFetchCollection[graph.GetIDer, groupsItemer]{}
|
_ data.BackupCollection = &lazyFetchCollection[graph.GetIDer, groupsItemer]{}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var errMetadataFilesNotSupported = clues.New("metadata files not supported")
|
||||||
|
|
||||||
const (
|
const (
|
||||||
collectionChannelBufferSize = 1000
|
collectionChannelBufferSize = 1000
|
||||||
numberOfRetries = 4
|
numberOfRetries = 4
|
||||||
@ -378,10 +380,20 @@ func (col *lazyFetchCollection[C, I]) streamItems(ctx context.Context, errs *fau
|
|||||||
|
|
||||||
// Handle metadata before data so that if metadata file fails,
|
// Handle metadata before data so that if metadata file fails,
|
||||||
// we are not left with an orphaned data file.
|
// we are not left with an orphaned data file.
|
||||||
|
//
|
||||||
|
// If the data download fails for some reason other than deleted in
|
||||||
|
// flight, we will still end up persisting a .meta file. This is
|
||||||
|
// fine however, since the next backup will overwrite it.
|
||||||
|
//
|
||||||
|
// If item is deleted in flight, we will end up with an orphaned
|
||||||
|
// .meta file. The only impact here is storage bloat, which
|
||||||
|
// is minimal. Other impact could be if we do an in-order restore
|
||||||
|
// using a tree built from .meta files. We may have some .meta
|
||||||
|
// files without corresponding .data files.
|
||||||
itemMeta, _, err := col.getAndAugment.getItemMetadata(
|
itemMeta, _, err := col.getAndAugment.getItemMetadata(
|
||||||
ictx,
|
ictx,
|
||||||
col.contains.container)
|
col.contains.container)
|
||||||
if err != nil && !errors.Is(err, metadata.ErrMetadataFilesNotSupported) {
|
if err != nil && !errors.Is(err, errMetadataFilesNotSupported) {
|
||||||
errs.AddRecoverable(ctx, clues.StackWC(ctx, err))
|
errs.AddRecoverable(ctx, clues.StackWC(ctx, err))
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|||||||
@ -27,7 +27,6 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/errs/core"
|
"github.com/alcionai/corso/src/pkg/errs/core"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type CollectionUnitSuite struct {
|
type CollectionUnitSuite struct {
|
||||||
@ -173,7 +172,7 @@ func (getAndAugmentChannelMessage) getItemMetadata(
|
|||||||
_ context.Context,
|
_ context.Context,
|
||||||
_ models.Channelable,
|
_ models.Channelable,
|
||||||
) (io.ReadCloser, int, error) {
|
) (io.ReadCloser, int, error) {
|
||||||
return nil, 0, metadata.ErrMetadataFilesNotSupported
|
return nil, 0, errMetadataFilesNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
//lint:ignore U1000 false linter issue due to generics
|
//lint:ignore U1000 false linter issue due to generics
|
||||||
@ -290,6 +289,7 @@ func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() {
|
|||||||
|
|
||||||
type getAndAugmentConversation struct {
|
type getAndAugmentConversation struct {
|
||||||
GetItemErr error
|
GetItemErr error
|
||||||
|
GetMetaErr error
|
||||||
CallIDs []string
|
CallIDs []string
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -313,8 +313,7 @@ func (m *getAndAugmentConversation) getItemMetadata(
|
|||||||
_ context.Context,
|
_ context.Context,
|
||||||
_ models.Conversationable,
|
_ models.Conversationable,
|
||||||
) (io.ReadCloser, int, error) {
|
) (io.ReadCloser, int, error) {
|
||||||
// Return some dummy data
|
return io.NopCloser(strings.NewReader("test")), 4, m.GetMetaErr
|
||||||
return io.NopCloser(strings.NewReader("test")), 4, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
|
|||||||
@ -10,11 +10,11 @@ import (
|
|||||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/collection/groups/metadata"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups"
|
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -12,8 +12,8 @@ import (
|
|||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
|
"github.com/alcionai/corso/src/internal/m365/collection/groups/metadata"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
metadata "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata/groups"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ConversationHandlerUnitSuite struct {
|
type ConversationHandlerUnitSuite struct {
|
||||||
|
|||||||
@ -1,13 +1,9 @@
|
|||||||
package metadata
|
package metadata
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/alcionai/clues"
|
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrMetadataFilesNotSupported = clues.New("metadata files not supported")
|
|
||||||
|
|
||||||
func IsMetadataFile(p path.Path) bool {
|
func IsMetadataFile(p path.Path) bool {
|
||||||
switch p.Service() {
|
switch p.Service() {
|
||||||
case path.OneDriveService:
|
case path.OneDriveService:
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user