From 85652bfd676161e70c9381654f2b2d20bb04d5ed Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Mon, 3 Jul 2023 12:51:08 -0700 Subject: [PATCH] Add Corso extensions package (#3729) Introduces interfaces for corso extensions, which is basically wrappers around `io.ReadCloser`. Will add integration in follow up PRs. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/pkg/backup/details/details.go | 5 + src/pkg/extensions/extensions.go | 61 ++++++++++ src/pkg/extensions/extensions_test.go | 169 ++++++++++++++++++++++++++ src/pkg/extensions/mock_extensions.go | 89 ++++++++++++++ 4 files changed, 324 insertions(+) create mode 100644 src/pkg/extensions/extensions.go create mode 100644 src/pkg/extensions/extensions_test.go create mode 100644 src/pkg/extensions/mock_extensions.go diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index f394d02b7..bd2dbb909 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -1044,3 +1044,8 @@ func updateFolderWithinDrive( return nil } + +// ExtensionInfo describes extension data associated with an item +type ExtensionInfo struct { + Data map[string]any `json:"data,omitempty"` +} diff --git a/src/pkg/extensions/extensions.go b/src/pkg/extensions/extensions.go new file mode 100644 index 000000000..3c6b1fc4c --- /dev/null +++ b/src/pkg/extensions/extensions.go @@ -0,0 +1,61 @@ +package extensions + +import ( + "context" + "io" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/logger" +) + +type CreateItemExtensioner interface { + CreateItemExtension( + context.Context, + io.ReadCloser, + details.ItemInfo, + *details.ExtensionInfo, + ) (io.ReadCloser, error) +} + +// AddItemExtensions wraps provided readcloser with extensions +// supplied via factory, with the first extension in slice being +// the innermost one. +func AddItemExtensions( + ctx context.Context, + rc io.ReadCloser, + info details.ItemInfo, + factories []CreateItemExtensioner, +) (io.ReadCloser, *details.ExtensionInfo, error) { + if rc == nil { + return nil, nil, clues.New("nil readcloser") + } + + if len(factories) == 0 { + return nil, nil, clues.New("no extensions supplied") + } + + ctx = clues.Add(ctx, "num_extensions", len(factories)) + + extInfo := &details.ExtensionInfo{ + Data: make(map[string]any), + } + + for _, factory := range factories { + if factory == nil { + return nil, nil, clues.New("nil extension factory") + } + + extRc, err := factory.CreateItemExtension(ctx, rc, info, extInfo) + if err != nil { + return nil, nil, clues.Wrap(err, "create item extension") + } + + rc = extRc + } + + logger.Ctx(ctx).Debug("added item extensions") + + return rc, extInfo, nil +} diff --git a/src/pkg/extensions/extensions_test.go b/src/pkg/extensions/extensions_test.go new file mode 100644 index 000000000..6a0eb7d1e --- /dev/null +++ b/src/pkg/extensions/extensions_test.go @@ -0,0 +1,169 @@ +package extensions + +// Tests for extensions.go + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup/details" +) + +type ExtensionsUnitSuite struct { + tester.Suite +} + +func TestExtensionsUnitSuite(t *testing.T) { + suite.Run(t, &ExtensionsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExtensionsUnitSuite) TestAddItemExtensions() { + type outputValidationFunc func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool + + var ( + testRc = io.NopCloser(bytes.NewReader([]byte("some data"))) + testItemInfo = details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + DriveID: "driveID", + }, + } + ) + + table := []struct { + name string + factories []CreateItemExtensioner + rc io.ReadCloser + validateOutputs outputValidationFunc + }{ + { + name: "happy path", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{}, + }, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err == nil && extRc != nil && extInfo != nil + }, + }, + { + name: "multiple valid factories", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{}, + &MockItemExtensionFactory{}, + }, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err == nil && extRc != nil && extInfo != nil + }, + }, + { + name: "no factories supplied", + factories: nil, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err != nil && extRc == nil && extInfo == nil + }, + }, + { + name: "factory slice contains nil", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{}, + nil, + &MockItemExtensionFactory{}, + }, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err != nil && extRc == nil && extInfo == nil + }, + }, + { + name: "factory call returns error", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{ + shouldReturnError: true, + }, + }, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err != nil && extRc == nil && extInfo == nil + }, + }, + { + name: "one or more factory calls return error", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{}, + &MockItemExtensionFactory{ + shouldReturnError: true, + }, + }, + rc: testRc, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err != nil && extRc == nil && extInfo == nil + }, + }, + { + name: "nil inner rc", + factories: []CreateItemExtensioner{ + &MockItemExtensionFactory{}, + }, + rc: nil, + validateOutputs: func( + extRc io.ReadCloser, + extInfo *details.ExtensionInfo, + err error, + ) bool { + return err != nil && extRc == nil && extInfo == nil + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ctx, flush := tester.NewContext(t) + defer flush() + + extRc, extInfo, err := AddItemExtensions( + ctx, + test.rc, + testItemInfo, + test.factories) + require.True(t, test.validateOutputs(extRc, extInfo, err)) + }) + } +} + +// TODO(pandeyabs): Tests to verify RC wrapper ordering by AddItemExtensioner diff --git a/src/pkg/extensions/mock_extensions.go b/src/pkg/extensions/mock_extensions.go new file mode 100644 index 000000000..3dda2f0d0 --- /dev/null +++ b/src/pkg/extensions/mock_extensions.go @@ -0,0 +1,89 @@ +package extensions + +import ( + "context" + "hash/crc32" + "io" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/logger" +) + +var _ io.ReadCloser = &MockExtension{} + +type MockExtension struct { + numBytes int + crc32 uint32 + info details.ItemInfo + extInfo *details.ExtensionInfo + innerRc io.ReadCloser + ctx context.Context + failOnRead bool + failOnClose bool +} + +func (me *MockExtension) Read(p []byte) (int, error) { + if me.failOnRead { + return 0, clues.New("mock read error") + } + + n, err := me.innerRc.Read(p) + if err != nil && err != io.EOF { + logger.CtxErr(me.ctx, err).Error("inner read error") + return n, err + } + + me.numBytes += n + me.crc32 = crc32.Update(me.crc32, crc32.IEEETable, p[:n]) + + if err == io.EOF { + logger.Ctx(me.ctx).Debug("mock extension reached EOF") + me.extInfo.Data["numBytes"] = me.numBytes + me.extInfo.Data["crc32"] = me.crc32 + } + + return n, err +} + +func (me *MockExtension) Close() error { + if me.failOnClose { + return clues.New("mock close error") + } + + err := me.innerRc.Close() + if err != nil { + return err + } + + me.extInfo.Data["numBytes"] = me.numBytes + me.extInfo.Data["crc32"] = me.crc32 + logger.Ctx(me.ctx).Infow( + "mock extension closed", + "numBytes", me.numBytes, "crc32", me.crc32) + + return nil +} + +type MockItemExtensionFactory struct { + shouldReturnError bool +} + +func (m *MockItemExtensionFactory) CreateItemExtension( + ctx context.Context, + rc io.ReadCloser, + info details.ItemInfo, + extInfo *details.ExtensionInfo, +) (io.ReadCloser, error) { + if m.shouldReturnError { + return nil, clues.New("factory error") + } + + return &MockExtension{ + ctx: ctx, + innerRc: rc, + info: info, + extInfo: extInfo, + }, nil +}