diff --git a/src/cli/config/account.go b/src/cli/config/account.go index 310ac97c3..45fd50058 100644 --- a/src/cli/config/account.go +++ b/src/cli/config/account.go @@ -6,7 +6,7 @@ import ( "github.com/alcionai/clues" "github.com/spf13/viper" - "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" ) @@ -64,7 +64,7 @@ func configureAccount( m365Cfg = account.M365Config{ M365: m365, - AzureTenantID: common.First( + AzureTenantID: str.First( overrides[account.AzureTenantID], m365Cfg.AzureTenantID, os.Getenv(account.AzureTenantID)), diff --git a/src/cli/config/storage.go b/src/cli/config/storage.go index 9aba1e5d9..af8dff397 100644 --- a/src/cli/config/storage.go +++ b/src/cli/config/storage.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/viper" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/storage" ) @@ -80,14 +81,14 @@ func configureStorage( } s3Cfg = storage.S3Config{ - Bucket: common.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)), - Endpoint: common.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)), - Prefix: common.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)), - DoNotUseTLS: common.ParseBool(common.First( + Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)), + Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)), + Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)), + DoNotUseTLS: str.ParseBool(str.First( overrides[storage.DoNotUseTLS], strconv.FormatBool(s3Cfg.DoNotUseTLS), os.Getenv(storage.PrefixKey))), - DoNotVerifyTLS: common.ParseBool(common.First( + DoNotVerifyTLS: str.ParseBool(str.First( overrides[storage.DoNotVerifyTLS], strconv.FormatBool(s3Cfg.DoNotVerifyTLS), os.Getenv(storage.PrefixKey))), diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index c267a828a..de5e61915 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -11,10 +11,10 @@ import ( "github.com/google/uuid" "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/connector" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/data" @@ -116,7 +116,7 @@ func getGCAndVerifyResourceOwner( idname.Provider, error, ) { - tid := common.First(Tenant, os.Getenv(account.AzureTenantID)) + tid := str.First(Tenant, os.Getenv(account.AzureTenantID)) if len(Tenant) == 0 { Tenant = tid diff --git a/src/cmd/getM365/exchange/get_item.go b/src/cmd/getM365/exchange/get_item.go index 1d644f97e..cc6e8cd6a 100644 --- a/src/cmd/getM365/exchange/get_item.go +++ b/src/cmd/getM365/exchange/get_item.go @@ -15,7 +15,8 @@ import ( "github.com/spf13/cobra" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/credentials" @@ -54,7 +55,7 @@ func handleExchangeCmd(cmd *cobra.Command, args []string) error { return nil } - tid := common.First(tenant, os.Getenv(account.AzureTenantID)) + tid := str.First(tenant, os.Getenv(account.AzureTenantID)) ctx := clues.Add( cmd.Context(), @@ -111,9 +112,7 @@ func runDisplayM365JSON( return err } - str := string(bs) - - err = sw.WriteStringValue("", &str) + err = sw.WriteStringValue("", ptr.To(string(bs))) if err != nil { return clues.Wrap(err, "Error writing string value: "+itemID) } diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go index ab1378ab9..414f50694 100644 --- a/src/cmd/getM365/onedrive/get_item.go +++ b/src/cmd/getM365/onedrive/get_item.go @@ -19,8 +19,8 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" @@ -57,7 +57,7 @@ func handleOneDriveCmd(cmd *cobra.Command, args []string) error { return nil } - tid := common.First(tenant, os.Getenv(account.AzureTenantID)) + tid := str.First(tenant, os.Getenv(account.AzureTenantID)) ctx := clues.Add( cmd.Context(), diff --git a/src/cmd/purge/purge.go b/src/cmd/purge/purge.go index d9f1133c1..337ea6f46 100644 --- a/src/cmd/purge/purge.go +++ b/src/cmd/purge/purge.go @@ -11,8 +11,8 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" @@ -263,7 +263,7 @@ func getGC(ctx context.Context) (account.Account, *connector.GraphConnector, err // get account info m365Cfg := account.M365Config{ M365: credentials.GetM365(), - AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)), + AzureTenantID: str.First(tenant, os.Getenv(account.AzureTenantID)), } acct, err := account.NewAccount(account.ProviderM365, m365Cfg) diff --git a/src/internal/common/slices.go b/src/internal/common/slices.go deleted file mode 100644 index 73c7c951e..000000000 --- a/src/internal/common/slices.go +++ /dev/null @@ -1,23 +0,0 @@ -package common - -// TODO: can be replaced with slices.Contains() -func ContainsString(super []string, sub string) bool { - for _, s := range super { - if s == sub { - return true - } - } - - return false -} - -// First returns the first non-zero valued string -func First(vs ...string) string { - for _, v := range vs { - if len(v) > 0 { - return v - } - } - - return "" -} diff --git a/src/internal/common/slices_test.go b/src/internal/common/slices_test.go deleted file mode 100644 index 9aae236a5..000000000 --- a/src/internal/common/slices_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package common_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/tester" -) - -type CommonSlicesSuite struct { - tester.Suite -} - -func TestCommonSlicesSuite(t *testing.T) { - s := &CommonSlicesSuite{Suite: tester.NewUnitSuite(t)} - suite.Run(t, s) -} - -func (suite *CommonSlicesSuite) TestContainsString() { - t := suite.T() - target := "fnords" - good := []string{"fnords"} - bad := []string{"foo", "bar"} - - assert.True(t, common.ContainsString(good, target)) - assert.False(t, common.ContainsString(bad, target)) -} diff --git a/src/internal/common/str/str.go b/src/internal/common/str/str.go new file mode 100644 index 000000000..9dcd46af8 --- /dev/null +++ b/src/internal/common/str/str.go @@ -0,0 +1,58 @@ +package str + +import ( + "fmt" + "strconv" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/ptr" +) + +// parseBool returns the bool value represented by the string +// or false on error +func ParseBool(v string) bool { + s, err := strconv.ParseBool(v) + if err != nil { + return false + } + + return s +} + +func FromMapToAny(k string, m map[string]any) (string, error) { + if len(m) == 0 { + return "", clues.New("missing entry").With("map_key", k) + } + + return FromAny(m[k]) +} + +func FromAny(a any) (string, error) { + if a == nil { + return "", clues.New("missing value") + } + + sp, ok := a.(*string) + if ok { + return ptr.Val(sp), nil + } + + s, ok := a.(string) + if ok { + return s, nil + } + + return "", clues.New(fmt.Sprintf("unexpected type: %T", a)) +} + +// First returns the first non-zero valued string +func First(vs ...string) string { + for _, v := range vs { + if len(v) > 0 { + return v + } + } + + return "" +} diff --git a/src/internal/common/strings.go b/src/internal/common/strings.go deleted file mode 100644 index e8db07c94..000000000 --- a/src/internal/common/strings.go +++ /dev/null @@ -1,14 +0,0 @@ -package common - -import "strconv" - -// parseBool returns the bool value represented by the string -// or false on error -func ParseBool(v string) bool { - s, err := strconv.ParseBool(v) - if err != nil { - return false - } - - return s -} diff --git a/src/internal/common/tform/tform.go b/src/internal/common/tform/tform.go new file mode 100644 index 000000000..64b43c316 --- /dev/null +++ b/src/internal/common/tform/tform.go @@ -0,0 +1,25 @@ +package tform + +import ( + "fmt" + + "github.com/alcionai/clues" +) + +func FromMapToAny[T any](k string, m map[string]any) (T, error) { + v, ok := m[k] + if !ok { + return *new(T), clues.New("entry not found") + } + + if v == nil { + return *new(T), clues.New("nil entry") + } + + vt, ok := v.(T) + if !ok { + return *new(T), clues.New(fmt.Sprintf("unexpected type: %T", v)) + } + + return vt, nil +} diff --git a/src/internal/connector/graph/cache_container.go b/src/internal/connector/graph/cache_container.go index fd9c06aac..1e3467639 100644 --- a/src/internal/connector/graph/cache_container.go +++ b/src/internal/connector/graph/cache_container.go @@ -169,21 +169,22 @@ func CreateCalendarDisplayable(entry any, parentID string) *CalendarDisplayable // helper funcs // ========================================= -// checkRequiredValues is a helper function to ensure that -// all the pointers are set prior to being called. -func CheckRequiredValues(c Container) error { - id, ok := ptr.ValOK(c.GetId()) - if !ok { +// CheckIDAndName is a validator that ensures the ID +// and name are populated and not zero valued. +func CheckIDAndName(c Container) error { + if c == nil { + return clues.New("nil container") + } + + id := ptr.Val(c.GetId()) + if len(id) == 0 { return clues.New("container missing ID") } - if _, ok := ptr.ValOK(c.GetDisplayName()); !ok { + dn := ptr.Val(c.GetDisplayName()) + if len(dn) == 0 { return clues.New("container missing display name").With("container_id", id) } - if _, ok := ptr.ValOK(c.GetParentFolderId()); !ok { - return clues.New("container missing parent ID").With("container_id", id) - } - return nil } diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index 39636a99c..f6af8ffce 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -14,7 +14,7 @@ import ( "go.uber.org/zap/zapcore" "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/str" ) // Default location for writing logs, initialized in platform specific files @@ -256,7 +256,7 @@ func (s Settings) EnsureDefaults() Settings { algs := []piiAlg{PIIPlainText, PIIMask, PIIHash} if len(set.PIIHandling) == 0 || !slices.Contains(algs, set.PIIHandling) { - set.PIIHandling = piiAlg(common.First(piiHandling, string(PIIPlainText))) + set.PIIHandling = piiAlg(str.First(piiHandling, string(PIIPlainText))) } if len(set.File) == 0 { diff --git a/src/pkg/services/m365/api/exchange_common.go b/src/pkg/services/m365/api/attachments.go similarity index 50% rename from src/pkg/services/m365/api/exchange_common.go rename to src/pkg/services/m365/api/attachments.go index 7f4f6afe2..e5125a64a 100644 --- a/src/pkg/services/m365/api/exchange_common.go +++ b/src/pkg/services/m365/api/attachments.go @@ -3,29 +3,11 @@ package api import ( "strings" - "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/graph" ) -// checkIDAndName is a helper function to ensure that -// the ID and name pointers are set prior to being called. -func checkIDAndName(c graph.Container) error { - id := ptr.Val(c.GetId()) - if len(id) == 0 { - return clues.New("container missing ID") - } - - dn := ptr.Val(c.GetDisplayName()) - if len(dn) == 0 { - return clues.New("container missing display name").With("container_id", id) - } - - return nil -} - func HasAttachments(body models.ItemBodyable) bool { if body == nil { return false diff --git a/src/pkg/services/m365/api/api.go b/src/pkg/services/m365/api/client.go similarity index 65% rename from src/pkg/services/m365/api/api.go rename to src/pkg/services/m365/api/client.go index 1500840fe..cf7930664 100644 --- a/src/pkg/services/m365/api/api.go +++ b/src/pkg/services/m365/api/client.go @@ -1,10 +1,7 @@ package api import ( - "context" - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" @@ -78,31 +75,3 @@ func newLargeItemService(creds account.M365Config) (*graph.Service, error) { return a, nil } - -// --------------------------------------------------------------------------- -// common types and consts -// --------------------------------------------------------------------------- - -// DeltaUpdate holds the results of a current delta token. It normally -// gets produced when aggregating the addition and removal of items in -// a delta-queryable folder. -type DeltaUpdate struct { - // the deltaLink itself - URL string - // true if the old delta was marked as invalid - Reset bool -} - -// GraphQuery represents functions which perform exchange-specific queries -// into M365 backstore. Responses -> returned items will only contain the information -// that is included in the options -// TODO: use selector or path for granularity into specific folders or specific date ranges -type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) - -// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve -// the default associated data of a M365 object. This varies by object. Additional -// Queries must be run to obtain the omitted fields. -type GraphRetrievalFunc func( - ctx context.Context, - user, m365ID string, -) (serialization.Parsable, error) diff --git a/src/pkg/services/m365/api/api_test.go b/src/pkg/services/m365/api/client_test.go similarity index 86% rename from src/pkg/services/m365/api/api_test.go rename to src/pkg/services/m365/api/client_test.go index cfaf8976f..196cc2322 100644 --- a/src/pkg/services/m365/api/api_test.go +++ b/src/pkg/services/m365/api/client_test.go @@ -49,41 +49,6 @@ func (suite *ExchangeServiceSuite) SetupSuite() { suite.gs = graph.NewService(adpt) } -func (suite *ExchangeServiceSuite) TestOptionsForCalendars() { - tests := []struct { - name string - params []string - checkError assert.ErrorAssertionFunc - }{ - { - name: "Empty Literal", - params: []string{}, - checkError: assert.NoError, - }, - { - name: "Invalid Parameter", - params: []string{"status"}, - checkError: assert.Error, - }, - { - name: "Invalid Parameters", - params: []string{"status", "height", "month"}, - checkError: assert.Error, - }, - { - name: "Valid Parameters", - params: []string{"changeKey", "events", "owner"}, - checkError: assert.NoError, - }, - } - for _, test := range tests { - suite.Run(test.name, func() { - _, err := optionsForCalendars(test.params) - test.checkError(suite.T(), err, clues.ToCore(err)) - }) - } -} - //nolint:lll var stubHTMLContent = "\r\n
Happy New Year,

In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture? 



Let me know if this meets our culture requirements.

Warm Regards,

Dustin
" diff --git a/src/pkg/services/m365/api/consts.go b/src/pkg/services/m365/api/consts.go new file mode 100644 index 000000000..0828d3b4d --- /dev/null +++ b/src/pkg/services/m365/api/consts.go @@ -0,0 +1,3 @@ +package api + +const maxPageSize = int32(999) diff --git a/src/pkg/services/m365/api/contacts.go b/src/pkg/services/m365/api/contacts.go index 2f5395b37..b958e2ea1 100644 --- a/src/pkg/services/m365/api/contacts.go +++ b/src/pkg/services/m365/api/contacts.go @@ -96,12 +96,13 @@ func (c Contacts) GetContainerByID( ctx context.Context, userID, dirID string, ) (graph.Container, error) { - ofcf, err := optionsForContactFolderByID([]string{"displayName", "parentFolderId"}) - if err != nil { - return nil, graph.Wrap(ctx, err, "setting contact folder options") + queryParams := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{ + Select: []string{"id", "displayName", "parentFolderId"}, + }, } - resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, ofcf) + resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, queryParams) if err != nil { return nil, graph.Stack(ctx, err) } @@ -125,11 +126,10 @@ func (c Contacts) EnumerateContainers( return graph.Stack(ctx, err) } - fields := []string{"displayName", "parentFolderId"} - - ofcf, err := optionsForContactChildFolders(fields) - if err != nil { - return graph.Wrap(ctx, err, "setting contact child folder options") + queryParams := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Select: []string{"id", "displayName", "parentFolderId"}, + }, } el := errs.Local() @@ -145,7 +145,7 @@ func (c Contacts) EnumerateContainers( break } - resp, err := builder.Get(ctx, ofcf) + resp, err := builder.Get(ctx, queryParams) if err != nil { return graph.Stack(ctx, err) } @@ -155,7 +155,7 @@ func (c Contacts) EnumerateContainers( return el.Failure() } - if err := checkIDAndName(fold); err != nil { + if err := graph.CheckIDAndName(fold); err != nil { errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation)) continue } @@ -200,28 +200,17 @@ func NewContactPager( gs graph.Servicer, user, directoryID string, immutableIDs bool, -) (itemPager, error) { - selecting, err := buildOptions([]string{"parentFolderId"}, fieldsForContacts) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - Headers: buildPreferHeaders(true, immutableIDs), - } - - if err != nil { - return &contactPager{}, err +) itemPager { + queryParams := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{ + Select: []string{"id", "parentFolderId"}, + }, + Headers: buildPreferHeaders(true, immutableIDs), } builder := gs.Client().Users().ByUserId(user).ContactFolders().ByContactFolderId(directoryID).Contacts() - return &contactPager{gs, builder, options}, nil + return &contactPager{gs, builder, queryParams} } func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { @@ -274,23 +263,12 @@ func NewContactDeltaPager( gs graph.Servicer, user, directoryID, deltaURL string, immutableIDs bool, -) (itemPager, error) { - selecting, err := buildOptions([]string{"parentFolderId"}, fieldsForContacts) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{ - Select: selecting, - } - +) itemPager { options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - Headers: buildPreferHeaders(true, immutableIDs), - } - - if err != nil { - return &contactDeltaPager{}, err + QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{ + Select: []string{"id", "parentFolderId"}, + }, + Headers: buildPreferHeaders(true, immutableIDs), } var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder @@ -300,7 +278,7 @@ func NewContactDeltaPager( builder = getContactDeltaBuilder(ctx, gs, user, directoryID, options) } - return &contactDeltaPager{gs, user, directoryID, builder, options}, nil + return &contactDeltaPager{gs, user, directoryID, builder, options} } func (p *contactDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { @@ -340,15 +318,8 @@ func (c Contacts) GetAddedAndRemovedItemIDs( "category", selectors.ExchangeContact, "container_id", directoryID) - pager, err := NewContactPager(ctx, service, user, directoryID, immutableIDs) - if err != nil { - return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating non-delta pager") - } - - deltaPager, err := NewContactDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs) - if err != nil { - return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager") - } + pager := NewContactPager(ctx, service, user, directoryID, immutableIDs) + deltaPager := NewContactDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs) return getAddedAndRemovedItemIDs(ctx, service, pager, deltaPager, oldDelta, canMakeDeltaQueries) } diff --git a/src/pkg/services/m365/api/delta.go b/src/pkg/services/m365/api/delta.go new file mode 100644 index 000000000..dc24961f0 --- /dev/null +++ b/src/pkg/services/m365/api/delta.go @@ -0,0 +1,11 @@ +package api + +// DeltaUpdate holds the results of a current delta token. It normally +// gets produced when aggregating the addition and removal of items in +// a delta-queryable folder. +type DeltaUpdate struct { + // the deltaLink itself + URL string + // true if the old delta was marked as invalid + Reset bool +} diff --git a/src/pkg/services/m365/api/drive.go b/src/pkg/services/m365/api/drive.go index 33c357033..8201f51e3 100644 --- a/src/pkg/services/m365/api/drive.go +++ b/src/pkg/services/m365/api/drive.go @@ -3,312 +3,14 @@ package api import ( "context" "fmt" - "strings" - "time" "github.com/alcionai/clues" - abstractions "github.com/microsoft/kiota-abstractions-go" "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/sites" - "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/graph/api" - onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts" - "github.com/alcionai/corso/src/pkg/logger" ) -func getValues[T any](l api.PageLinker) ([]T, error) { - page, ok := l.(interface{ GetValue() []T }) - if !ok { - return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l)) - } - - return page.GetValue(), nil -} - -// max we can do is 999 -const pageSize = int32(999) - -type driveItemPager struct { - gs graph.Servicer - driveID string - builder *drives.ItemItemsItemDeltaRequestBuilder - options *drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration -} - -func NewItemPager( - gs graph.Servicer, - driveID, link string, - fields []string, -) *driveItemPager { - pageCount := pageSize - - headers := abstractions.NewRequestHeaders() - preferHeaderItems := []string{ - "deltashowremovedasdeleted", - "deltatraversepermissiongaps", - "deltashowsharingchanges", - "hierarchicalsharing", - } - headers.Add("Prefer", strings.Join(preferHeaderItems, ",")) - - requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{ - Headers: headers, - QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{ - Top: &pageCount, - Select: fields, - }, - } - - res := &driveItemPager{ - gs: gs, - driveID: driveID, - options: requestConfig, - builder: gs.Client(). - Drives(). - ByDriveId(driveID). - Items().ByDriveItemId(onedrive.RootID).Delta(), - } - - if len(link) > 0 { - res.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, gs.Adapter()) - } - - return res -} - -func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) { - var ( - resp api.DeltaPageLinker - err error - ) - - resp, err = p.builder.Get(ctx, p.options) - if err != nil { - return nil, graph.Stack(ctx, err) - } - - return resp, nil -} - -func (p *driveItemPager) SetNext(link string) { - p.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, p.gs.Adapter()) -} - -func (p *driveItemPager) Reset() { - p.builder = p.gs.Client(). - Drives(). - ByDriveId(p.driveID). - Items(). - ByDriveItemId(onedrive.RootID). - Delta() -} - -func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) { - return getValues[models.DriveItemable](l) -} - -type userDrivePager struct { - userID string - gs graph.Servicer - builder *users.ItemDrivesRequestBuilder - options *users.ItemDrivesRequestBuilderGetRequestConfiguration -} - -func NewUserDrivePager( - gs graph.Servicer, - userID string, - fields []string, -) *userDrivePager { - requestConfig := &users.ItemDrivesRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemDrivesRequestBuilderGetQueryParameters{ - Select: fields, - }, - } - - res := &userDrivePager{ - userID: userID, - gs: gs, - options: requestConfig, - builder: gs.Client().Users().ByUserId(userID).Drives(), - } - - return res -} - -type nopUserDrivePageLinker struct { - drive models.Driveable -} - -func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil } - -func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { - var ( - resp api.PageLinker - err error - ) - - d, err := p.gs.Client().Users().ByUserId(p.userID).Drive().Get(ctx, nil) - if err != nil { - return nil, graph.Stack(ctx, err) - } - - resp = &nopUserDrivePageLinker{drive: d} - - // TODO(keepers): turn back on when we can separate drive enumeration - // from default drive lookup. - - // resp, err = p.builder.Get(ctx, p.options) - // if err != nil { - // return nil, graph.Stack(ctx, err) - // } - - return resp, nil -} - -func (p *userDrivePager) SetNext(link string) { - p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) -} - -func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { - nl, ok := l.(*nopUserDrivePageLinker) - if !ok || nl == nil { - return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l)) - } - - // TODO(keepers): turn back on when we can separate drive enumeration - // from default drive lookup. - - // return getValues[models.Driveable](l) - - return []models.Driveable{nl.drive}, nil -} - -type siteDrivePager struct { - gs graph.Servicer - builder *sites.ItemDrivesRequestBuilder - options *sites.ItemDrivesRequestBuilderGetRequestConfiguration -} - -// NewSiteDrivePager is a constructor for creating a siteDrivePager -// fields are the associated site drive fields that are desired to be returned -// in a query. NOTE: Fields are case-sensitive. Incorrect field settings will -// cause errors during later paging. -// Available fields: https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0 -func NewSiteDrivePager( - gs graph.Servicer, - siteID string, - fields []string, -) *siteDrivePager { - requestConfig := &sites.ItemDrivesRequestBuilderGetRequestConfiguration{ - QueryParameters: &sites.ItemDrivesRequestBuilderGetQueryParameters{ - Select: fields, - }, - } - - res := &siteDrivePager{ - gs: gs, - options: requestConfig, - builder: gs.Client().Sites().BySiteId(siteID).Drives(), - } - - return res -} - -func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { - var ( - resp api.PageLinker - err error - ) - - resp, err = p.builder.Get(ctx, p.options) - if err != nil { - return nil, graph.Stack(ctx, err) - } - - return resp, nil -} - -func (p *siteDrivePager) SetNext(link string) { - p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) -} - -func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { - return getValues[models.Driveable](l) -} - -// DrivePager pages through different types of drive owners -type DrivePager interface { - GetPage(context.Context) (api.PageLinker, error) - SetNext(nextLink string) - ValuesIn(api.PageLinker) ([]models.Driveable, error) -} - -// GetAllDrives fetches all drives for the given pager -func GetAllDrives( - ctx context.Context, - pager DrivePager, - retry bool, - maxRetryCount int, -) ([]models.Driveable, error) { - ds := []models.Driveable{} - - if !retry { - maxRetryCount = 0 - } - - // Loop through all pages returned by Graph API. - for { - var ( - err error - page api.PageLinker - ) - - // Retry Loop for Drive retrieval. Request can timeout - for i := 0; i <= maxRetryCount; i++ { - page, err = pager.GetPage(ctx) - if err != nil { - if clues.HasLabel(err, graph.LabelsMysiteNotFound) { - logger.Ctx(ctx).Infof("resource owner does not have a drive") - return make([]models.Driveable, 0), nil // no license or drives. - } - - if graph.IsErrTimeout(err) && i < maxRetryCount { - time.Sleep(time.Duration(3*(i+1)) * time.Second) - continue - } - - return nil, graph.Wrap(ctx, err, "retrieving drives") - } - - // No error encountered, break the retry loop so we can extract results - // and see if there's another page to fetch. - break - } - - tmp, err := pager.ValuesIn(page) - if err != nil { - return nil, graph.Wrap(ctx, err, "extracting drives from response") - } - - ds = append(ds, tmp...) - - nextLink := ptr.Val(page.GetOdataNextLink()) - if len(nextLink) == 0 { - break - } - - pager.SetNext(nextLink) - } - - logger.Ctx(ctx).Debugf("retrieved %d valid drives", len(ds)) - - return ds, nil -} - // generic drive item getter func GetDriveItem( ctx context.Context, diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go new file mode 100644 index 000000000..684277fd6 --- /dev/null +++ b/src/pkg/services/m365/api/drive_pager.go @@ -0,0 +1,325 @@ +package api + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/alcionai/clues" + abstractions "github.com/microsoft/kiota-abstractions-go" + "github.com/microsoftgraph/msgraph-sdk-go/drives" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/sites" + "github.com/microsoftgraph/msgraph-sdk-go/users" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" + onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts" + "github.com/alcionai/corso/src/pkg/logger" +) + +// --------------------------------------------------------------------------- +// item pager +// --------------------------------------------------------------------------- + +type driveItemPager struct { + gs graph.Servicer + driveID string + builder *drives.ItemItemsItemDeltaRequestBuilder + options *drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration +} + +func NewItemPager( + gs graph.Servicer, + driveID, link string, + fields []string, +) *driveItemPager { + headers := abstractions.NewRequestHeaders() + preferHeaderItems := []string{ + "deltashowremovedasdeleted", + "deltatraversepermissiongaps", + "deltashowsharingchanges", + "hierarchicalsharing", + } + headers.Add("Prefer", strings.Join(preferHeaderItems, ",")) + + requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{ + Headers: headers, + QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{ + Top: ptr.To(maxPageSize), + Select: fields, + }, + } + + res := &driveItemPager{ + gs: gs, + driveID: driveID, + options: requestConfig, + builder: gs.Client(). + Drives(). + ByDriveId(driveID). + Items().ByDriveItemId(onedrive.RootID).Delta(), + } + + if len(link) > 0 { + res.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, gs.Adapter()) + } + + return res +} + +func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) { + var ( + resp api.DeltaPageLinker + err error + ) + + resp, err = p.builder.Get(ctx, p.options) + if err != nil { + return nil, graph.Stack(ctx, err) + } + + return resp, nil +} + +func (p *driveItemPager) SetNext(link string) { + p.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, p.gs.Adapter()) +} + +func (p *driveItemPager) Reset() { + p.builder = p.gs.Client(). + Drives(). + ByDriveId(p.driveID). + Items(). + ByDriveItemId(onedrive.RootID). + Delta() +} + +func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) { + return getValues[models.DriveItemable](l) +} + +// --------------------------------------------------------------------------- +// user pager +// --------------------------------------------------------------------------- + +type userDrivePager struct { + userID string + gs graph.Servicer + builder *users.ItemDrivesRequestBuilder + options *users.ItemDrivesRequestBuilderGetRequestConfiguration +} + +func NewUserDrivePager( + gs graph.Servicer, + userID string, + fields []string, +) *userDrivePager { + requestConfig := &users.ItemDrivesRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemDrivesRequestBuilderGetQueryParameters{ + Select: fields, + }, + } + + res := &userDrivePager{ + userID: userID, + gs: gs, + options: requestConfig, + builder: gs.Client().Users().ByUserId(userID).Drives(), + } + + return res +} + +type nopUserDrivePageLinker struct { + drive models.Driveable +} + +func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil } + +func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { + var ( + resp api.PageLinker + err error + ) + + d, err := p.gs.Client().Users().ByUserId(p.userID).Drive().Get(ctx, nil) + if err != nil { + return nil, graph.Stack(ctx, err) + } + + resp = &nopUserDrivePageLinker{drive: d} + + // TODO(keepers): turn back on when we can separate drive enumeration + // from default drive lookup. + + // resp, err = p.builder.Get(ctx, p.options) + // if err != nil { + // return nil, graph.Stack(ctx, err) + // } + + return resp, nil +} + +func (p *userDrivePager) SetNext(link string) { + p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) +} + +func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { + nl, ok := l.(*nopUserDrivePageLinker) + if !ok || nl == nil { + return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l)) + } + + // TODO(keepers): turn back on when we can separate drive enumeration + // from default drive lookup. + + // return getValues[models.Driveable](l) + + return []models.Driveable{nl.drive}, nil +} + +// --------------------------------------------------------------------------- +// site pager +// --------------------------------------------------------------------------- + +type siteDrivePager struct { + gs graph.Servicer + builder *sites.ItemDrivesRequestBuilder + options *sites.ItemDrivesRequestBuilderGetRequestConfiguration +} + +// NewSiteDrivePager is a constructor for creating a siteDrivePager +// fields are the associated site drive fields that are desired to be returned +// in a query. NOTE: Fields are case-sensitive. Incorrect field settings will +// cause errors during later paging. +// Available fields: https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0 +func NewSiteDrivePager( + gs graph.Servicer, + siteID string, + fields []string, +) *siteDrivePager { + requestConfig := &sites.ItemDrivesRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemDrivesRequestBuilderGetQueryParameters{ + Select: fields, + }, + } + + res := &siteDrivePager{ + gs: gs, + options: requestConfig, + builder: gs.Client().Sites().BySiteId(siteID).Drives(), + } + + return res +} + +func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { + var ( + resp api.PageLinker + err error + ) + + resp, err = p.builder.Get(ctx, p.options) + if err != nil { + return nil, graph.Stack(ctx, err) + } + + return resp, nil +} + +func (p *siteDrivePager) SetNext(link string) { + p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) +} + +func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { + return getValues[models.Driveable](l) +} + +// --------------------------------------------------------------------------- +// drive pager +// --------------------------------------------------------------------------- + +// DrivePager pages through different types of drive owners +type DrivePager interface { + GetPage(context.Context) (api.PageLinker, error) + SetNext(nextLink string) + ValuesIn(api.PageLinker) ([]models.Driveable, error) +} + +// GetAllDrives fetches all drives for the given pager +func GetAllDrives( + ctx context.Context, + pager DrivePager, + retry bool, + maxRetryCount int, +) ([]models.Driveable, error) { + ds := []models.Driveable{} + + if !retry { + maxRetryCount = 0 + } + + // Loop through all pages returned by Graph API. + for { + var ( + err error + page api.PageLinker + ) + + // Retry Loop for Drive retrieval. Request can timeout + for i := 0; i <= maxRetryCount; i++ { + page, err = pager.GetPage(ctx) + if err != nil { + if clues.HasLabel(err, graph.LabelsMysiteNotFound) { + logger.Ctx(ctx).Infof("resource owner does not have a drive") + return make([]models.Driveable, 0), nil // no license or drives. + } + + if graph.IsErrTimeout(err) && i < maxRetryCount { + time.Sleep(time.Duration(3*(i+1)) * time.Second) + continue + } + + return nil, graph.Wrap(ctx, err, "retrieving drives") + } + + // No error encountered, break the retry loop so we can extract results + // and see if there's another page to fetch. + break + } + + tmp, err := pager.ValuesIn(page) + if err != nil { + return nil, graph.Wrap(ctx, err, "extracting drives from response") + } + + ds = append(ds, tmp...) + + nextLink := ptr.Val(page.GetOdataNextLink()) + if len(nextLink) == 0 { + break + } + + pager.SetNext(nextLink) + } + + logger.Ctx(ctx).Debugf("retrieved %d valid drives", len(ds)) + + return ds, nil +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func getValues[T any](l api.PageLinker) ([]T, error) { + page, ok := l.(interface{ GetValue() []T }) + if !ok { + return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l)) + } + + return page.GetValue(), nil +} diff --git a/src/pkg/services/m365/api/events.go b/src/pkg/services/m365/api/events.go index 3d84a9c53..5cccdc49b 100644 --- a/src/pkg/services/m365/api/events.go +++ b/src/pkg/services/m365/api/events.go @@ -84,12 +84,13 @@ func (c Events) GetContainerByID( return nil, graph.Stack(ctx, err) } - ofc, err := optionsForCalendarsByID([]string{"name", "owner"}) - if err != nil { - return nil, graph.Wrap(ctx, err, "setting event calendar options") + queryParams := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ + Select: []string{"id", "name", "owner"}, + }, } - cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, ofc) + cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, queryParams) if err != nil { return nil, graph.Stack(ctx, err).WithClues(ctx) } @@ -129,7 +130,7 @@ func (c Events) GetContainerByName( cal := resp.GetValue()[0] cd := CalendarDisplayable{Calendarable: cal} - if err := checkIDAndName(cd); err != nil { + if err := graph.CheckIDAndName(cd); err != nil { return nil, err } @@ -199,9 +200,10 @@ func (c Events) EnumerateContainers( return graph.Stack(ctx, err) } - ofc, err := optionsForCalendars([]string{"name"}) - if err != nil { - return graph.Wrap(ctx, err, "setting calendar options") + queryParams := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{ + Select: []string{"id", "name"}, + }, } el := errs.Local() @@ -212,7 +214,7 @@ func (c Events) EnumerateContainers( break } - resp, err := builder.Get(ctx, ofc) + resp, err := builder.Get(ctx, queryParams) if err != nil { return graph.Stack(ctx, err) } @@ -223,7 +225,7 @@ func (c Events) EnumerateContainers( } cd := CalendarDisplayable{Calendarable: cal} - if err := checkIDAndName(cd); err != nil { + if err := graph.CheckIDAndName(cd); err != nil { errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation)) continue } diff --git a/src/pkg/services/m365/api/shared.go b/src/pkg/services/m365/api/item_pager.go similarity index 100% rename from src/pkg/services/m365/api/shared.go rename to src/pkg/services/m365/api/item_pager.go diff --git a/src/pkg/services/m365/api/shared_test.go b/src/pkg/services/m365/api/item_pager_test.go similarity index 100% rename from src/pkg/services/m365/api/shared_test.go rename to src/pkg/services/m365/api/item_pager_test.go diff --git a/src/pkg/services/m365/api/mail.go b/src/pkg/services/m365/api/mail.go index 2abb889c0..c951027be 100644 --- a/src/pkg/services/m365/api/mail.go +++ b/src/pkg/services/m365/api/mail.go @@ -123,9 +123,10 @@ func (c Mail) GetContainerByID( return nil, graph.Stack(ctx, err) } - ofmf, err := optionsForMailFoldersItem([]string{"displayName", "parentFolderId"}) - if err != nil { - return nil, graph.Wrap(ctx, err, "setting mail folder options") + queryParams := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{ + Select: []string{"id", "displayName", "parentFolderId"}, + }, } resp, err := service.Client(). @@ -133,7 +134,7 @@ func (c Mail) GetContainerByID( ByUserId(userID). MailFolders(). ByMailFolderId(dirID). - Get(ctx, ofmf) + Get(ctx, queryParams) if err != nil { return nil, graph.Stack(ctx, err) } @@ -380,23 +381,12 @@ func NewMailPager( gs graph.Servicer, user, directoryID string, immutableIDs bool, -) (itemPager, error) { - selecting, err := buildOptions([]string{"isRead"}, fieldsForMessages) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - Headers: buildPreferHeaders(true, immutableIDs), - } - - if err != nil { - return &mailPager{}, err +) itemPager { + queryParams := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{ + Select: []string{"id", "isRead"}, + }, + Headers: buildPreferHeaders(true, immutableIDs), } builder := gs.Client(). @@ -406,7 +396,7 @@ func NewMailPager( ByMailFolderId(directoryID). Messages() - return &mailPager{gs, builder, options}, nil + return &mailPager{gs, builder, queryParams} } func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { @@ -466,23 +456,12 @@ func NewMailDeltaPager( gs graph.Servicer, user, directoryID, oldDelta string, immutableIDs bool, -) (itemPager, error) { - selecting, err := buildOptions([]string{"isRead"}, fieldsForMessages) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - Headers: buildPreferHeaders(true, immutableIDs), - } - - if err != nil { - return &mailDeltaPager{}, err +) itemPager { + queryParams := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{ + Select: []string{"id", "isRead"}, + }, + Headers: buildPreferHeaders(true, immutableIDs), } var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder @@ -490,10 +469,10 @@ func NewMailDeltaPager( if len(oldDelta) > 0 { builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()) } else { - builder = getMailDeltaBuilder(ctx, gs, user, directoryID, options) + builder = getMailDeltaBuilder(ctx, gs, user, directoryID, queryParams) } - return &mailDeltaPager{gs, user, directoryID, builder, options}, nil + return &mailDeltaPager{gs, user, directoryID, builder, queryParams} } func (p *mailDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { @@ -539,15 +518,8 @@ func (c Mail) GetAddedAndRemovedItemIDs( "category", selectors.ExchangeMail, "container_id", directoryID) - pager, err := NewMailPager(ctx, service, user, directoryID, immutableIDs) - if err != nil { - return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager") - } - - deltaPager, err := NewMailDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs) - if err != nil { - return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager") - } + pager := NewMailPager(ctx, service, user, directoryID, immutableIDs) + deltaPager := NewMailDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs) return getAddedAndRemovedItemIDs(ctx, service, pager, deltaPager, oldDelta, canMakeDeltaQueries) } diff --git a/src/pkg/services/m365/api/options.go b/src/pkg/services/m365/api/options.go deleted file mode 100644 index ff506e7d5..000000000 --- a/src/pkg/services/m365/api/options.go +++ /dev/null @@ -1,214 +0,0 @@ -package api - -import ( - "fmt" - "strings" - - "github.com/alcionai/clues" - abstractions "github.com/microsoft/kiota-abstractions-go" - "github.com/microsoftgraph/msgraph-sdk-go/users" -) - -// ----------------------------------------------------------------------- -// Constant Section -// Defines the allowable strings that can be passed into -// selectors for M365 objects -// ----------------------------------------------------------------------- -var ( - fieldsForCalendars = map[string]struct{}{ - "changeKey": {}, - "events": {}, - "id": {}, - "isDefaultCalendar": {}, - "name": {}, - "owner": {}, - } - - fieldsForFolders = map[string]struct{}{ - "childFolderCount": {}, - "displayName": {}, - "id": {}, - "isHidden": {}, - "parentFolderId": {}, - "totalItemCount": {}, - "unreadItemCount": {}, - } - - fieldsForMessages = map[string]struct{}{ - "conservationId": {}, - "conversationIndex": {}, - "parentFolderId": {}, - "subject": {}, - "webLink": {}, - "id": {}, - "isRead": {}, - } - - fieldsForContacts = map[string]struct{}{ - "id": {}, - "companyName": {}, - "department": {}, - "displayName": {}, - "fileAs": {}, - "givenName": {}, - "manager": {}, - "parentFolderId": {}, - } -) - -const ( - // headerKeyPrefer is used to set query preferences - headerKeyPrefer = "Prefer" - // maxPageSizeHeaderFmt is used to indicate max page size - // preferences - maxPageSizeHeaderFmt = "odata.maxpagesize=%d" - // deltaMaxPageSize is the max page size to use for delta queries - deltaMaxPageSize = 200 - idTypeFmt = "IdType=%q" - immutableIDType = "ImmutableId" -) - -// ----------------------------------------------------------------------- -// exchange.Query Option Section -// These functions can be used to filter a response on M365 -// Graph queries and reduce / filter the amount of data returned -// which reduces the overall latency of complex calls -// ----------------------------------------------------------------------- - -// optionsForCalendars places allowed options for exchange.Calendar object -// @param moreOps should reflect elements from fieldsForCalendars -// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler -func optionsForCalendars(moreOps []string) ( - *users.ItemCalendarsRequestBuilderGetRequestConfiguration, - error, -) { - selecting, err := buildOptions(moreOps, fieldsForCalendars) - if err != nil { - return nil, err - } - // should be a CalendarsRequestBuilderGetRequestConfiguration - requestParams := &users.ItemCalendarsRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParams, - } - - return options, nil -} - -// optionsForCalendarsByID places allowed options for exchange.Calendar object -// @param moreOps should reflect elements from fieldsForCalendars -// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler -func optionsForCalendarsByID(moreOps []string) ( - *users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration, - error, -) { - selecting, err := buildOptions(moreOps, fieldsForCalendars) - if err != nil { - return nil, err - } - // should be a CalendarsRequestBuilderGetRequestConfiguration - requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParams, - } - - return options, nil -} - -func optionsForContactFolderByID(moreOps []string) ( - *users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration, - error, -) { - selecting, err := buildOptions(moreOps, fieldsForFolders) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - -// optionsForMailFoldersItem transforms the options into a more dynamic call for MailFoldersById. -// moreOps is a []string of options(e.g. "displayName", "isHidden") -// Returns first call in MailFoldersById().GetWithRequestConfigurationAndResponseHandler(options, handler) -func optionsForMailFoldersItem( - moreOps []string, -) (*users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) { - selecting, err := buildOptions(moreOps, fieldsForFolders) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - -// optionsForContactChildFolders builds a contacts child folders request. -func optionsForContactChildFolders( - moreOps []string, -) (*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) { - selecting, err := buildOptions(moreOps, fieldsForContacts) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - -// buildOptions - Utility Method for verifying if select options are valid for the m365 object type -// @return is a pair. The first is a string literal of allowable options based on the object type, -// the second is an error. An error is returned if an unsupported option or optionIdentifier was used -func buildOptions(fields []string, allowed map[string]struct{}) ([]string, error) { - returnedOptions := []string{"id"} - - for _, entry := range fields { - _, ok := allowed[entry] - if !ok { - return nil, clues.New("unsupported field: " + entry) - } - } - - return append(returnedOptions, fields...), nil -} - -// buildPreferHeaders returns the headers we add to item delta page -// requests. -func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders { - var allHeaders []string - - if pageSize { - allHeaders = append(allHeaders, fmt.Sprintf(maxPageSizeHeaderFmt, deltaMaxPageSize)) - } - - if immutableID { - allHeaders = append(allHeaders, fmt.Sprintf(idTypeFmt, immutableIDType)) - } - - headers := abstractions.NewRequestHeaders() - headers.Add(headerKeyPrefer, strings.Join(allHeaders, ",")) - - return headers -} diff --git a/src/pkg/services/m365/api/query_params.go b/src/pkg/services/m365/api/query_params.go new file mode 100644 index 000000000..68eff9962 --- /dev/null +++ b/src/pkg/services/m365/api/query_params.go @@ -0,0 +1,26 @@ +package api + +import ( + "fmt" + "strings" + + abstractions "github.com/microsoft/kiota-abstractions-go" +) + +// buildPreferHeaders returns the headers we add to item delta page requests. +func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders { + var allHeaders []string + + if pageSize { + allHeaders = append(allHeaders, fmt.Sprintf("odata.maxpagesize=%d", maxPageSize)) + } + + if immutableID { + allHeaders = append(allHeaders, `IdType="ImmutableId"`) + } + + headers := abstractions.NewRequestHeaders() + headers.Add("Prefer", strings.Join(allHeaders, ",")) + + return headers +} diff --git a/src/pkg/services/m365/api/users.go b/src/pkg/services/m365/api/users.go index 6bba52de5..108b558bc 100644 --- a/src/pkg/services/m365/api/users.go +++ b/src/pkg/services/m365/api/users.go @@ -14,6 +14,8 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" + "github.com/alcionai/corso/src/internal/common/tform" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" @@ -394,90 +396,90 @@ func (c Users) getMailboxSettings( additionalData := settings.GetAdditionalData() - mi.ArchiveFolder, err = toString(ctx, "archiveFolder", additionalData) + mi.ArchiveFolder, err = str.FromMapToAny("archiveFolder", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.Timezone, err = toString(ctx, "timeZone", additionalData) + mi.Timezone, err = str.FromMapToAny("timeZone", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.DateFormat, err = toString(ctx, "dateFormat", additionalData) + mi.DateFormat, err = str.FromMapToAny("dateFormat", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.TimeFormat, err = toString(ctx, "timeFormat", additionalData) + mi.TimeFormat, err = str.FromMapToAny("timeFormat", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.Purpose, err = toString(ctx, "userPurpose", additionalData) + mi.Purpose, err = str.FromMapToAny("userPurpose", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.DelegateMeetMsgDeliveryOpt, err = toString(ctx, "delegateMeetingMessageDeliveryOptions", additionalData) + mi.DelegateMeetMsgDeliveryOpt, err = str.FromMapToAny("delegateMeetingMessageDeliveryOptions", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) // decode automatic replies settings - replySetting, err := toT[map[string]any](ctx, "automaticRepliesSetting", additionalData) + replySetting, err := tform.FromMapToAny[map[string]any]("automaticRepliesSetting", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.Status, err = toString(ctx, "status", replySetting) + mi.AutomaticRepliesSetting.Status, err = str.FromMapToAny("status", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ExternalAudience, err = toString(ctx, "externalAudience", replySetting) + mi.AutomaticRepliesSetting.ExternalAudience, err = str.FromMapToAny("externalAudience", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ExternalReplyMessage, err = toString(ctx, "externalReplyMessage", replySetting) + mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.FromMapToAny("externalReplyMessage", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.InternalReplyMessage, err = toString(ctx, "internalReplyMessage", replySetting) + mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.FromMapToAny("internalReplyMessage", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) // decode scheduledStartDateTime - startDateTime, err := toT[map[string]any](ctx, "scheduledStartDateTime", replySetting) + startDateTime, err := tform.FromMapToAny[map[string]any]("scheduledStartDateTime", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = toString(ctx, "dateTime", startDateTime) + mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.FromMapToAny("dateTime", startDateTime) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = toString(ctx, "timeZone", startDateTime) + mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.FromMapToAny("timeZone", startDateTime) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - endDateTime, err := toT[map[string]any](ctx, "scheduledEndDateTime", replySetting) + endDateTime, err := tform.FromMapToAny[map[string]any]("scheduledEndDateTime", replySetting) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = toString(ctx, "dateTime", endDateTime) + mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.FromMapToAny("dateTime", endDateTime) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = toString(ctx, "timeZone", endDateTime) + mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.FromMapToAny("timeZone", endDateTime) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) // Language decode - language, err := toT[map[string]any](ctx, "language", additionalData) + language, err := tform.FromMapToAny[map[string]any]("language", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.Language.DisplayName, err = toString(ctx, "displayName", language) + mi.Language.DisplayName, err = str.FromMapToAny("displayName", language) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.Language.Locale, err = toString(ctx, "locale", language) + mi.Language.Locale, err = str.FromMapToAny("locale", language) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) // working hours - workingHours, err := toT[map[string]any](ctx, "workingHours", additionalData) + workingHours, err := tform.FromMapToAny[map[string]any]("workingHours", additionalData) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.WorkingHours.StartTime, err = toString(ctx, "startTime", workingHours) + mi.WorkingHours.StartTime, err = str.FromMapToAny("startTime", workingHours) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.WorkingHours.EndTime, err = toString(ctx, "endTime", workingHours) + mi.WorkingHours.EndTime, err = str.FromMapToAny("endTime", workingHours) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - timeZone, err := toT[map[string]any](ctx, "timeZone", workingHours) + timeZone, err := tform.FromMapToAny[map[string]any]("timeZone", workingHours) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - mi.WorkingHours.TimeZone.Name, err = toString(ctx, "name", timeZone) + mi.WorkingHours.TimeZone.Name, err = str.FromMapToAny("name", timeZone) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) - days, err := toT[[]any](ctx, "daysOfWeek", workingHours) + days, err := tform.FromMapToAny[[]any]("daysOfWeek", workingHours) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) for _, day := range days { - s, err := anyToString(ctx, "dayOfTheWeek", day) + s, err := str.FromAny(day) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.WorkingHours.DaysOfWeek = append(mi.WorkingHours.DaysOfWeek, s) } @@ -510,53 +512,3 @@ func validateUser(item models.Userable) error { return nil } - -func toString(ctx context.Context, key string, data map[string]any) (string, error) { - ctx = clues.Add(ctx, "setting_name", key) - - if len(data) == 0 { - logger.Ctx(ctx).Info("not found: ", key) - return "", ErrMailBoxSettingsNotFound - } - - return anyToString(ctx, key, data[key]) -} - -func anyToString(ctx context.Context, key string, val any) (string, error) { - if val == nil { - logger.Ctx(ctx).Info("nil value: ", key) - return "", ErrMailBoxSettingsNotFound - } - - sp, ok := val.(*string) - if !ok { - logger.Ctx(ctx).Info("value is not a *string: ", key) - return "", ErrMailBoxSettingsNotFound - } - - return ptr.Val(sp), nil -} - -func toT[T any](ctx context.Context, key string, data map[string]any) (T, error) { - ctx = clues.Add(ctx, "setting_name", key) - - if len(data) == 0 { - logger.Ctx(ctx).Info("not found: ", key) - return *new(T), ErrMailBoxSettingsNotFound - } - - val := data[key] - - if data == nil { - logger.Ctx(ctx).Info("nil value: ", key) - return *new(T), ErrMailBoxSettingsNotFound - } - - value, ok := val.(T) - if !ok { - logger.Ctx(ctx).Info(fmt.Sprintf("unexpected type for %s: %T", key, val)) - return *new(T), ErrMailBoxSettingsNotFound - } - - return value, nil -} diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index baa497b71..17fe89f02 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -6,6 +6,7 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/str" ) type S3Config struct { @@ -68,8 +69,8 @@ func (s Storage) S3Config() (S3Config, error) { c.Bucket = orEmptyString(s.Config[keyS3Bucket]) c.Endpoint = orEmptyString(s.Config[keyS3Endpoint]) c.Prefix = orEmptyString(s.Config[keyS3Prefix]) - c.DoNotUseTLS = common.ParseBool(s.Config[keyS3DoNotUseTLS]) - c.DoNotVerifyTLS = common.ParseBool(s.Config[keyS3DoNotVerifyTLS]) + c.DoNotUseTLS = str.ParseBool(s.Config[keyS3DoNotUseTLS]) + c.DoNotVerifyTLS = str.ParseBool(s.Config[keyS3DoNotVerifyTLS]) } return c, c.validate()