larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor api options (#3428)

Browse Source 
now that exchange api has been folded in with the rest of the m365 api, it doesn't make sense to maintain an options file with only exchange functionality.  Since all calls in the file were used 1:1 with some api func, those options have been moved into their respective api funcs.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #1996

#### Test Plan

- [x] 💚 E2E
This commit is contained in:
Keepers 2023-05-18 12:49:44 -06:00 committed by GitHub
parent cc35b1ed97
commit 6b7745745a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 571 additions and 887 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli/config/account.go
View File

@ -6,7 +6,7 @@ import (
"github.com/alcionai/clues"
"github.com/spf13/viper"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
)
@ -64,7 +64,7 @@ func configureAccount(
m365Cfg = account.M365Config{
M365: m365,
AzureTenantID: common.First(
AzureTenantID: str.First(
overrides[account.AzureTenantID],
m365Cfg.AzureTenantID,
os.Getenv(account.AzureTenantID)),

11
src/cli/config/storage.go
View File

@ -10,6 +10,7 @@ import (
"github.com/spf13/viper"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -80,14 +81,14 @@ func configureStorage(
}
s3Cfg = storage.S3Config{
Bucket: common.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)),
Endpoint: common.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)),
Prefix: common.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)),
DoNotUseTLS: common.ParseBool(common.First(
Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)),
Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)),
Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)),
DoNotUseTLS: str.ParseBool(str.First(
overrides[storage.DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS),
os.Getenv(storage.PrefixKey))),
DoNotVerifyTLS: common.ParseBool(common.First(
DoNotVerifyTLS: str.ParseBool(str.First(
overrides[storage.DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
os.Getenv(storage.PrefixKey))),

4
src/cmd/factory/impl/common.go
View File

@ -11,10 +11,10 @@ import (
"github.com/google/uuid"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/connector"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
"github.com/alcionai/corso/src/internal/data"
@ -116,7 +116,7 @@ func getGCAndVerifyResourceOwner(
idname.Provider,
error,
) {
tid := common.First(Tenant, os.Getenv(account.AzureTenantID))
tid := str.First(Tenant, os.Getenv(account.AzureTenantID))
if len(Tenant) == 0 {
Tenant = tid

9
src/cmd/getM365/exchange/get_item.go
View File

@ -15,7 +15,8 @@ import (
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/credentials"
@ -54,7 +55,7 @@ func handleExchangeCmd(cmd *cobra.Command, args []string) error {
return nil
}
tid := common.First(tenant, os.Getenv(account.AzureTenantID))
tid := str.First(tenant, os.Getenv(account.AzureTenantID))
ctx := clues.Add(
cmd.Context(),
@ -111,9 +112,7 @@ func runDisplayM365JSON(
return err
}
str := string(bs)
err = sw.WriteStringValue("", &str)
err = sw.WriteStringValue("", ptr.To(string(bs)))
if err != nil {
return clues.Wrap(err, "Error writing string value: "+itemID)
}

4
src/cmd/getM365/onedrive/get_item.go
View File

@ -19,8 +19,8 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
@ -57,7 +57,7 @@ func handleOneDriveCmd(cmd *cobra.Command, args []string) error {
return nil
}
tid := common.First(tenant, os.Getenv(account.AzureTenantID))
tid := str.First(tenant, os.Getenv(account.AzureTenantID))
ctx := clues.Add(
cmd.Context(),

4
src/cmd/purge/purge.go
View File

@ -11,8 +11,8 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
@ -263,7 +263,7 @@ func getGC(ctx context.Context) (account.Account, *connector.GraphConnector, err
// get account info
m365Cfg := account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)),
AzureTenantID: str.First(tenant, os.Getenv(account.AzureTenantID)),
}
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)

23
src/internal/common/slices.go
View File

@ -1,23 +0,0 @@
package common
// TODO: can be replaced with slices.Contains()
func ContainsString(super []string, sub string) bool {
for _, s := range super {
if s == sub {
return true
}
}
return false
}
// First returns the first non-zero valued string
func First(vs ...string) string {
for _, v := range vs {
if len(v) > 0 {
return v
}
}
return ""
}

30
src/internal/common/slices_test.go
View File

@ -1,30 +0,0 @@
package common_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
)
type CommonSlicesSuite struct {
tester.Suite
}
func TestCommonSlicesSuite(t *testing.T) {
s := &CommonSlicesSuite{Suite: tester.NewUnitSuite(t)}
suite.Run(t, s)
}
func (suite *CommonSlicesSuite) TestContainsString() {
t := suite.T()
target := "fnords"
good := []string{"fnords"}
bad := []string{"foo", "bar"}
assert.True(t, common.ContainsString(good, target))
assert.False(t, common.ContainsString(bad, target))
}

58
src/internal/common/str/str.go Normal file
View File

@ -0,0 +1,58 @@
package str
import (
"fmt"
"strconv"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
)
// parseBool returns the bool value represented by the string
// or false on error
func ParseBool(v string) bool {
s, err := strconv.ParseBool(v)
if err != nil {
return false
}
return s
}
func FromMapToAny(k string, m map[string]any) (string, error) {
if len(m) == 0 {
return "", clues.New("missing entry").With("map_key", k)
}
return FromAny(m[k])
}
func FromAny(a any) (string, error) {
if a == nil {
return "", clues.New("missing value")
}
sp, ok := a.(*string)
if ok {
return ptr.Val(sp), nil
}
s, ok := a.(string)
if ok {
return s, nil
}
return "", clues.New(fmt.Sprintf("unexpected type: %T", a))
}
// First returns the first non-zero valued string
func First(vs ...string) string {
for _, v := range vs {
if len(v) > 0 {
return v
}
}
return ""
}

14
src/internal/common/strings.go
View File

@ -1,14 +0,0 @@
package common
import "strconv"
// parseBool returns the bool value represented by the string
// or false on error
func ParseBool(v string) bool {
s, err := strconv.ParseBool(v)
if err != nil {
return false
}
return s
}

25
src/internal/common/tform/tform.go Normal file
View File

@ -0,0 +1,25 @@
package tform
import (
"fmt"
"github.com/alcionai/clues"
)
func FromMapToAny[T any](k string, m map[string]any) (T, error) {
v, ok := m[k]
if !ok {
return *new(T), clues.New("entry not found")
}
if v == nil {
return *new(T), clues.New("nil entry")
}
vt, ok := v.(T)
if !ok {
return *new(T), clues.New(fmt.Sprintf("unexpected type: %T", v))
}
return vt, nil
}

21
src/internal/connector/graph/cache_container.go
View File

@ -169,21 +169,22 @@ func CreateCalendarDisplayable(entry any, parentID string) *CalendarDisplayable
// helper funcs
// =========================================
// checkRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func CheckRequiredValues(c Container) error {
id, ok := ptr.ValOK(c.GetId())
if !ok {
// CheckIDAndName is a validator that ensures the ID
// and name are populated and not zero valued.
func CheckIDAndName(c Container) error {
if c == nil {
return clues.New("nil container")
}
id := ptr.Val(c.GetId())
if len(id) == 0 {
return clues.New("container missing ID")
}
if _, ok := ptr.ValOK(c.GetDisplayName()); !ok {
dn := ptr.Val(c.GetDisplayName())
if len(dn) == 0 {
return clues.New("container missing display name").With("container_id", id)
}
if _, ok := ptr.ValOK(c.GetParentFolderId()); !ok {
return clues.New("container missing parent ID").With("container_id", id)
}
return nil
}

4
src/pkg/logger/logger.go
View File

@ -14,7 +14,7 @@ import (
"go.uber.org/zap/zapcore"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
)
// Default location for writing logs, initialized in platform specific files
@ -256,7 +256,7 @@ func (s Settings) EnsureDefaults() Settings {
algs := []piiAlg{PIIPlainText, PIIMask, PIIHash}
if len(set.PIIHandling) == 0 || !slices.Contains(algs, set.PIIHandling) {
set.PIIHandling = piiAlg(common.First(piiHandling, string(PIIPlainText)))
set.PIIHandling = piiAlg(str.First(piiHandling, string(PIIPlainText)))
}
if len(set.File) == 0 {

18
src/pkg/services/m365/api/exchange_common.go → src/pkg/services/m365/api/attachments.go
View File

@ -3,29 +3,11 @@ package api
import (
"strings"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
)
// checkIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func checkIDAndName(c graph.Container) error {
id := ptr.Val(c.GetId())
if len(id) == 0 {
return clues.New("container missing ID")
}
dn := ptr.Val(c.GetDisplayName())
if len(dn) == 0 {
return clues.New("container missing display name").With("container_id", id)
}
return nil
}
func HasAttachments(body models.ItemBodyable) bool {
if body == nil {
return false

31
src/pkg/services/m365/api/api.go → src/pkg/services/m365/api/client.go
View File

@ -1,10 +1,7 @@
package api
import (
"context"
"github.com/alcionai/clues"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
@ -78,31 +75,3 @@ func newLargeItemService(creds account.M365Config) (*graph.Service, error) {
return a, nil
}
// ---------------------------------------------------------------------------
// common types and consts
// ---------------------------------------------------------------------------
// DeltaUpdate holds the results of a current delta token. It normally
// gets produced when aggregating the addition and removal of items in
// a delta-queryable folder.
type DeltaUpdate struct {
// the deltaLink itself
URL string
// true if the old delta was marked as invalid
Reset bool
}
// GraphQuery represents functions which perform exchange-specific queries
// into M365 backstore. Responses -> returned items will only contain the information
// that is included in the options
// TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error)
// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve
// the default associated data of a M365 object. This varies by object. Additional
// Queries must be run to obtain the omitted fields.
type GraphRetrievalFunc func(
ctx context.Context,
user, m365ID string,
) (serialization.Parsable, error)

35
src/pkg/services/m365/api/api_test.go → src/pkg/services/m365/api/client_test.go
View File

@ -49,41 +49,6 @@ func (suite *ExchangeServiceSuite) SetupSuite() {
suite.gs = graph.NewService(adpt)
}
func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
_, err := optionsForCalendars(test.params)
test.checkError(suite.T(), err, clues.ToCore(err))
})
}
}
//nolint:lll
var stubHTMLContent = "<html><head>\r\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"><style type=\"text/css\" style=\"display:none\">\r\n<!--\r\np\r\n\t{margin-top:0;\r\n\tmargin-bottom:0}\r\n-->\r\n</style></head><body dir=\"ltr\"><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Happy New Year,</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture?&nbsp;</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><img class=\"FluidPluginCopy ContentPasted0 w-2070 h-1380\" size=\"5854817\" data-outlook-trace=\"F:1|T:1\" src=\"cid:85f4faa3-9851-40c7-ba0a-e63dce1185f9\" style=\"max-width:100%\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Let me know if this meets our culture requirements.</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Warm Regards,</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Dustin</div></body></html>"

3
src/pkg/services/m365/api/consts.go Normal file
View File

@ -0,0 +1,3 @@
package api
const maxPageSize = int32(999)

81
src/pkg/services/m365/api/contacts.go
View File

@ -96,12 +96,13 @@ func (c Contacts) GetContainerByID(
ctx context.Context,
userID, dirID string,
) (graph.Container, error) {
ofcf, err := optionsForContactFolderByID([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, graph.Wrap(ctx, err, "setting contact folder options")
queryParams := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
},
}
resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, ofcf)
resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, queryParams)
if err != nil {
return nil, graph.Stack(ctx, err)
}
@ -125,11 +126,10 @@ func (c Contacts) EnumerateContainers(
return graph.Stack(ctx, err)
}
fields := []string{"displayName", "parentFolderId"}
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return graph.Wrap(ctx, err, "setting contact child folder options")
queryParams := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
},
}
el := errs.Local()
@ -145,7 +145,7 @@ func (c Contacts) EnumerateContainers(
break
}
resp, err := builder.Get(ctx, ofcf)
resp, err := builder.Get(ctx, queryParams)
if err != nil {
return graph.Stack(ctx, err)
}
@ -155,7 +155,7 @@ func (c Contacts) EnumerateContainers(
return el.Failure()
}
if err := checkIDAndName(fold); err != nil {
if err := graph.CheckIDAndName(fold); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue
}
@ -200,28 +200,17 @@ func NewContactPager(
gs graph.Servicer,
user, directoryID string,
immutableIDs bool,
) (itemPager, error) {
selecting, err := buildOptions([]string{"parentFolderId"}, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
Headers: buildPreferHeaders(true, immutableIDs),
}
if err != nil {
return &contactPager{}, err
) itemPager {
queryParams := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"},
},
Headers: buildPreferHeaders(true, immutableIDs),
}
builder := gs.Client().Users().ByUserId(user).ContactFolders().ByContactFolderId(directoryID).Contacts()
return &contactPager{gs, builder, options}, nil
return &contactPager{gs, builder, queryParams}
}
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
@ -274,23 +263,12 @@ func NewContactDeltaPager(
gs graph.Servicer,
user, directoryID, deltaURL string,
immutableIDs bool,
) (itemPager, error) {
selecting, err := buildOptions([]string{"parentFolderId"}, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
) itemPager {
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
Headers: buildPreferHeaders(true, immutableIDs),
}
if err != nil {
return &contactDeltaPager{}, err
QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"},
},
Headers: buildPreferHeaders(true, immutableIDs),
}
var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder
@ -300,7 +278,7 @@ func NewContactDeltaPager(
builder = getContactDeltaBuilder(ctx, gs, user, directoryID, options)
}
return &contactDeltaPager{gs, user, directoryID, builder, options}, nil
return &contactDeltaPager{gs, user, directoryID, builder, options}
}
func (p *contactDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
@ -340,15 +318,8 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
"category", selectors.ExchangeContact,
"container_id", directoryID)
pager, err := NewContactPager(ctx, service, user, directoryID, immutableIDs)
if err != nil {
return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating non-delta pager")
}
deltaPager, err := NewContactDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs)
if err != nil {
return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager")
}
pager := NewContactPager(ctx, service, user, directoryID, immutableIDs)
deltaPager := NewContactDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs)
return getAddedAndRemovedItemIDs(ctx, service, pager, deltaPager, oldDelta, canMakeDeltaQueries)
}

11
src/pkg/services/m365/api/delta.go Normal file
View File

@ -0,0 +1,11 @@
package api
// DeltaUpdate holds the results of a current delta token. It normally
// gets produced when aggregating the addition and removal of items in
// a delta-queryable folder.
type DeltaUpdate struct {
// the deltaLink itself
URL string
// true if the old delta was marked as invalid
Reset bool
}

298
src/pkg/services/m365/api/drive.go
View File

@ -3,312 +3,14 @@ package api
import (
"context"
"fmt"
"strings"
"time"
"github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
"github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/logger"
)
func getValues[T any](l api.PageLinker) ([]T, error) {
page, ok := l.(interface{ GetValue() []T })
if !ok {
return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l))
}
return page.GetValue(), nil
}
// max we can do is 999
const pageSize = int32(999)
type driveItemPager struct {
gs graph.Servicer
driveID string
builder *drives.ItemItemsItemDeltaRequestBuilder
options *drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration
}
func NewItemPager(
gs graph.Servicer,
driveID, link string,
fields []string,
) *driveItemPager {
pageCount := pageSize
headers := abstractions.NewRequestHeaders()
preferHeaderItems := []string{
"deltashowremovedasdeleted",
"deltatraversepermissiongaps",
"deltashowsharingchanges",
"hierarchicalsharing",
}
headers.Add("Prefer", strings.Join(preferHeaderItems, ","))
requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{
Headers: headers,
QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{
Top: &pageCount,
Select: fields,
},
}
res := &driveItemPager{
gs: gs,
driveID: driveID,
options: requestConfig,
builder: gs.Client().
Drives().
ByDriveId(driveID).
Items().ByDriveItemId(onedrive.RootID).Delta(),
}
if len(link) > 0 {
res.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, gs.Adapter())
}
return res
}
func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) {
var (
resp api.DeltaPageLinker
err error
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
func (p *driveItemPager) SetNext(link string) {
p.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, p.gs.Adapter())
}
func (p *driveItemPager) Reset() {
p.builder = p.gs.Client().
Drives().
ByDriveId(p.driveID).
Items().
ByDriveItemId(onedrive.RootID).
Delta()
}
func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) {
return getValues[models.DriveItemable](l)
}
type userDrivePager struct {
userID string
gs graph.Servicer
builder *users.ItemDrivesRequestBuilder
options *users.ItemDrivesRequestBuilderGetRequestConfiguration
}
func NewUserDrivePager(
gs graph.Servicer,
userID string,
fields []string,
) *userDrivePager {
requestConfig := &users.ItemDrivesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemDrivesRequestBuilderGetQueryParameters{
Select: fields,
},
}
res := &userDrivePager{
userID: userID,
gs: gs,
options: requestConfig,
builder: gs.Client().Users().ByUserId(userID).Drives(),
}
return res
}
type nopUserDrivePageLinker struct {
drive models.Driveable
}
func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil }
func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
var (
resp api.PageLinker
err error
)
d, err := p.gs.Client().Users().ByUserId(p.userID).Drive().Get(ctx, nil)
if err != nil {
return nil, graph.Stack(ctx, err)
}
resp = &nopUserDrivePageLinker{drive: d}
// TODO(keepers): turn back on when we can separate drive enumeration
// from default drive lookup.
// resp, err = p.builder.Get(ctx, p.options)
// if err != nil {
// return nil, graph.Stack(ctx, err)
// }
return resp, nil
}
func (p *userDrivePager) SetNext(link string) {
p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
nl, ok := l.(*nopUserDrivePageLinker)
if !ok || nl == nil {
return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l))
}
// TODO(keepers): turn back on when we can separate drive enumeration
// from default drive lookup.
// return getValues[models.Driveable](l)
return []models.Driveable{nl.drive}, nil
}
type siteDrivePager struct {
gs graph.Servicer
builder *sites.ItemDrivesRequestBuilder
options *sites.ItemDrivesRequestBuilderGetRequestConfiguration
}
// NewSiteDrivePager is a constructor for creating a siteDrivePager
// fields are the associated site drive fields that are desired to be returned
// in a query. NOTE: Fields are case-sensitive. Incorrect field settings will
// cause errors during later paging.
// Available fields: https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0
func NewSiteDrivePager(
gs graph.Servicer,
siteID string,
fields []string,
) *siteDrivePager {
requestConfig := &sites.ItemDrivesRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.ItemDrivesRequestBuilderGetQueryParameters{
Select: fields,
},
}
res := &siteDrivePager{
gs: gs,
options: requestConfig,
builder: gs.Client().Sites().BySiteId(siteID).Drives(),
}
return res
}
func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
var (
resp api.PageLinker
err error
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
func (p *siteDrivePager) SetNext(link string) {
p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
return getValues[models.Driveable](l)
}
// DrivePager pages through different types of drive owners
type DrivePager interface {
GetPage(context.Context) (api.PageLinker, error)
SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error)
}
// GetAllDrives fetches all drives for the given pager
func GetAllDrives(
ctx context.Context,
pager DrivePager,
retry bool,
maxRetryCount int,
) ([]models.Driveable, error) {
ds := []models.Driveable{}
if !retry {
maxRetryCount = 0
}
// Loop through all pages returned by Graph API.
for {
var (
err error
page api.PageLinker
)
// Retry Loop for Drive retrieval. Request can timeout
for i := 0; i <= maxRetryCount; i++ {
page, err = pager.GetPage(ctx)
if err != nil {
if clues.HasLabel(err, graph.LabelsMysiteNotFound) {
logger.Ctx(ctx).Infof("resource owner does not have a drive")
return make([]models.Driveable, 0), nil // no license or drives.
}
if graph.IsErrTimeout(err) && i < maxRetryCount {
time.Sleep(time.Duration(3*(i+1)) * time.Second)
continue
}
return nil, graph.Wrap(ctx, err, "retrieving drives")
}
// No error encountered, break the retry loop so we can extract results
// and see if there's another page to fetch.
break
}
tmp, err := pager.ValuesIn(page)
if err != nil {
return nil, graph.Wrap(ctx, err, "extracting drives from response")
}
ds = append(ds, tmp...)
nextLink := ptr.Val(page.GetOdataNextLink())
if len(nextLink) == 0 {
break
}
pager.SetNext(nextLink)
}
logger.Ctx(ctx).Debugf("retrieved %d valid drives", len(ds))
return ds, nil
}
// generic drive item getter
func GetDriveItem(
ctx context.Context,

325
src/pkg/services/m365/api/drive_pager.go Normal file
View File

@ -0,0 +1,325 @@
package api
import (
"context"
"fmt"
"strings"
"time"
"github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
"github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/logger"
)
// ---------------------------------------------------------------------------
// item pager
// ---------------------------------------------------------------------------
type driveItemPager struct {
gs graph.Servicer
driveID string
builder *drives.ItemItemsItemDeltaRequestBuilder
options *drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration
}
func NewItemPager(
gs graph.Servicer,
driveID, link string,
fields []string,
) *driveItemPager {
headers := abstractions.NewRequestHeaders()
preferHeaderItems := []string{
"deltashowremovedasdeleted",
"deltatraversepermissiongaps",
"deltashowsharingchanges",
"hierarchicalsharing",
}
headers.Add("Prefer", strings.Join(preferHeaderItems, ","))
requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{
Headers: headers,
QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{
Top: ptr.To(maxPageSize),
Select: fields,
},
}
res := &driveItemPager{
gs: gs,
driveID: driveID,
options: requestConfig,
builder: gs.Client().
Drives().
ByDriveId(driveID).
Items().ByDriveItemId(onedrive.RootID).Delta(),
}
if len(link) > 0 {
res.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, gs.Adapter())
}
return res
}
func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) {
var (
resp api.DeltaPageLinker
err error
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
func (p *driveItemPager) SetNext(link string) {
p.builder = drives.NewItemItemsItemDeltaRequestBuilder(link, p.gs.Adapter())
}
func (p *driveItemPager) Reset() {
p.builder = p.gs.Client().
Drives().
ByDriveId(p.driveID).
Items().
ByDriveItemId(onedrive.RootID).
Delta()
}
func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) {
return getValues[models.DriveItemable](l)
}
// ---------------------------------------------------------------------------
// user pager
// ---------------------------------------------------------------------------
type userDrivePager struct {
userID string
gs graph.Servicer
builder *users.ItemDrivesRequestBuilder
options *users.ItemDrivesRequestBuilderGetRequestConfiguration
}
func NewUserDrivePager(
gs graph.Servicer,
userID string,
fields []string,
) *userDrivePager {
requestConfig := &users.ItemDrivesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemDrivesRequestBuilderGetQueryParameters{
Select: fields,
},
}
res := &userDrivePager{
userID: userID,
gs: gs,
options: requestConfig,
builder: gs.Client().Users().ByUserId(userID).Drives(),
}
return res
}
type nopUserDrivePageLinker struct {
drive models.Driveable
}
func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil }
func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
var (
resp api.PageLinker
err error
)
d, err := p.gs.Client().Users().ByUserId(p.userID).Drive().Get(ctx, nil)
if err != nil {
return nil, graph.Stack(ctx, err)
}
resp = &nopUserDrivePageLinker{drive: d}
// TODO(keepers): turn back on when we can separate drive enumeration
// from default drive lookup.
// resp, err = p.builder.Get(ctx, p.options)
// if err != nil {
// return nil, graph.Stack(ctx, err)
// }
return resp, nil
}
func (p *userDrivePager) SetNext(link string) {
p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
nl, ok := l.(*nopUserDrivePageLinker)
if !ok || nl == nil {
return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l))
}
// TODO(keepers): turn back on when we can separate drive enumeration
// from default drive lookup.
// return getValues[models.Driveable](l)
return []models.Driveable{nl.drive}, nil
}
// ---------------------------------------------------------------------------
// site pager
// ---------------------------------------------------------------------------
type siteDrivePager struct {
gs graph.Servicer
builder *sites.ItemDrivesRequestBuilder
options *sites.ItemDrivesRequestBuilderGetRequestConfiguration
}
// NewSiteDrivePager is a constructor for creating a siteDrivePager
// fields are the associated site drive fields that are desired to be returned
// in a query. NOTE: Fields are case-sensitive. Incorrect field settings will
// cause errors during later paging.
// Available fields: https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0
func NewSiteDrivePager(
gs graph.Servicer,
siteID string,
fields []string,
) *siteDrivePager {
requestConfig := &sites.ItemDrivesRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.ItemDrivesRequestBuilderGetQueryParameters{
Select: fields,
},
}
res := &siteDrivePager{
gs: gs,
options: requestConfig,
builder: gs.Client().Sites().BySiteId(siteID).Drives(),
}
return res
}
func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
var (
resp api.PageLinker
err error
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
func (p *siteDrivePager) SetNext(link string) {
p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
return getValues[models.Driveable](l)
}
// ---------------------------------------------------------------------------
// drive pager
// ---------------------------------------------------------------------------
// DrivePager pages through different types of drive owners
type DrivePager interface {
GetPage(context.Context) (api.PageLinker, error)
SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error)
}
// GetAllDrives fetches all drives for the given pager
func GetAllDrives(
ctx context.Context,
pager DrivePager,
retry bool,
maxRetryCount int,
) ([]models.Driveable, error) {
ds := []models.Driveable{}
if !retry {
maxRetryCount = 0
}
// Loop through all pages returned by Graph API.
for {
var (
err error
page api.PageLinker
)
// Retry Loop for Drive retrieval. Request can timeout
for i := 0; i <= maxRetryCount; i++ {
page, err = pager.GetPage(ctx)
if err != nil {
if clues.HasLabel(err, graph.LabelsMysiteNotFound) {
logger.Ctx(ctx).Infof("resource owner does not have a drive")
return make([]models.Driveable, 0), nil // no license or drives.
}
if graph.IsErrTimeout(err) && i < maxRetryCount {
time.Sleep(time.Duration(3*(i+1)) * time.Second)
continue
}
return nil, graph.Wrap(ctx, err, "retrieving drives")
}
// No error encountered, break the retry loop so we can extract results
// and see if there's another page to fetch.
break
}
tmp, err := pager.ValuesIn(page)
if err != nil {
return nil, graph.Wrap(ctx, err, "extracting drives from response")
}
ds = append(ds, tmp...)
nextLink := ptr.Val(page.GetOdataNextLink())
if len(nextLink) == 0 {
break
}
pager.SetNext(nextLink)
}
logger.Ctx(ctx).Debugf("retrieved %d valid drives", len(ds))
return ds, nil
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
func getValues[T any](l api.PageLinker) ([]T, error) {
page, ok := l.(interface{ GetValue() []T })
if !ok {
return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l))
}
return page.GetValue(), nil
}

22
src/pkg/services/m365/api/events.go
View File

@ -84,12 +84,13 @@ func (c Events) GetContainerByID(
return nil, graph.Stack(ctx, err)
}
ofc, err := optionsForCalendarsByID([]string{"name", "owner"})
if err != nil {
return nil, graph.Wrap(ctx, err, "setting event calendar options")
queryParams := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{
Select: []string{"id", "name", "owner"},
},
}
cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, ofc)
cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, queryParams)
if err != nil {
return nil, graph.Stack(ctx, err).WithClues(ctx)
}
@ -129,7 +130,7 @@ func (c Events) GetContainerByName(
cal := resp.GetValue()[0]
cd := CalendarDisplayable{Calendarable: cal}
if err := checkIDAndName(cd); err != nil {
if err := graph.CheckIDAndName(cd); err != nil {
return nil, err
}
@ -199,9 +200,10 @@ func (c Events) EnumerateContainers(
return graph.Stack(ctx, err)
}
ofc, err := optionsForCalendars([]string{"name"})
if err != nil {
return graph.Wrap(ctx, err, "setting calendar options")
queryParams := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{
Select: []string{"id", "name"},
},
}
el := errs.Local()
@ -212,7 +214,7 @@ func (c Events) EnumerateContainers(
break
}
resp, err := builder.Get(ctx, ofc)
resp, err := builder.Get(ctx, queryParams)
if err != nil {
return graph.Stack(ctx, err)
}
@ -223,7 +225,7 @@ func (c Events) EnumerateContainers(
}
cd := CalendarDisplayable{Calendarable: cal}
if err := checkIDAndName(cd); err != nil {
if err := graph.CheckIDAndName(cd); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue
}

0
src/pkg/services/m365/api/shared.go → src/pkg/services/m365/api/item_pager.go
View File

0
src/pkg/services/m365/api/shared_test.go → src/pkg/services/m365/api/item_pager_test.go
View File

72
src/pkg/services/m365/api/mail.go
View File

@ -123,9 +123,10 @@ func (c Mail) GetContainerByID(
return nil, graph.Stack(ctx, err)
}
ofmf, err := optionsForMailFoldersItem([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, graph.Wrap(ctx, err, "setting mail folder options")
queryParams := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
},
}
resp, err := service.Client().
@ -133,7 +134,7 @@ func (c Mail) GetContainerByID(
ByUserId(userID).
MailFolders().
ByMailFolderId(dirID).
Get(ctx, ofmf)
Get(ctx, queryParams)
if err != nil {
return nil, graph.Stack(ctx, err)
}
@ -380,23 +381,12 @@ func NewMailPager(
gs graph.Servicer,
user, directoryID string,
immutableIDs bool,
) (itemPager, error) {
selecting, err := buildOptions([]string{"isRead"}, fieldsForMessages)
if err != nil {
return nil, err
}
requestParameters := &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
Headers: buildPreferHeaders(true, immutableIDs),
}
if err != nil {
return &mailPager{}, err
) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"},
},
Headers: buildPreferHeaders(true, immutableIDs),
}
builder := gs.Client().
@ -406,7 +396,7 @@ func NewMailPager(
ByMailFolderId(directoryID).
Messages()
return &mailPager{gs, builder, options}, nil
return &mailPager{gs, builder, queryParams}
}
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
@ -466,23 +456,12 @@ func NewMailDeltaPager(
gs graph.Servicer,
user, directoryID, oldDelta string,
immutableIDs bool,
) (itemPager, error) {
selecting, err := buildOptions([]string{"isRead"}, fieldsForMessages)
if err != nil {
return nil, err
}
requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
Headers: buildPreferHeaders(true, immutableIDs),
}
if err != nil {
return &mailDeltaPager{}, err
) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"},
},
Headers: buildPreferHeaders(true, immutableIDs),
}
var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder
@ -490,10 +469,10 @@ func NewMailDeltaPager(
if len(oldDelta) > 0 {
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())
} else {
builder = getMailDeltaBuilder(ctx, gs, user, directoryID, options)
builder = getMailDeltaBuilder(ctx, gs, user, directoryID, queryParams)
}
return &mailDeltaPager{gs, user, directoryID, builder, options}, nil
return &mailDeltaPager{gs, user, directoryID, builder, queryParams}
}
func (p *mailDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
@ -539,15 +518,8 @@ func (c Mail) GetAddedAndRemovedItemIDs(
"category", selectors.ExchangeMail,
"container_id", directoryID)
pager, err := NewMailPager(ctx, service, user, directoryID, immutableIDs)
if err != nil {
return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager")
}
deltaPager, err := NewMailDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs)
if err != nil {
return nil, nil, DeltaUpdate{}, graph.Wrap(ctx, err, "creating delta pager")
}
pager := NewMailPager(ctx, service, user, directoryID, immutableIDs)
deltaPager := NewMailDeltaPager(ctx, service, user, directoryID, oldDelta, immutableIDs)
return getAddedAndRemovedItemIDs(ctx, service, pager, deltaPager, oldDelta, canMakeDeltaQueries)
}

214
src/pkg/services/m365/api/options.go
View File

@ -1,214 +0,0 @@
package api
import (
"fmt"
"strings"
"github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
"github.com/microsoftgraph/msgraph-sdk-go/users"
)
// -----------------------------------------------------------------------
// Constant Section
// Defines the allowable strings that can be passed into
// selectors for M365 objects
// -----------------------------------------------------------------------
var (
fieldsForCalendars = map[string]struct{}{
"changeKey": {},
"events": {},
"id": {},
"isDefaultCalendar": {},
"name": {},
"owner": {},
}
fieldsForFolders = map[string]struct{}{
"childFolderCount": {},
"displayName": {},
"id": {},
"isHidden": {},
"parentFolderId": {},
"totalItemCount": {},
"unreadItemCount": {},
}
fieldsForMessages = map[string]struct{}{
"conservationId": {},
"conversationIndex": {},
"parentFolderId": {},
"subject": {},
"webLink": {},
"id": {},
"isRead": {},
}
fieldsForContacts = map[string]struct{}{
"id": {},
"companyName": {},
"department": {},
"displayName": {},
"fileAs": {},
"givenName": {},
"manager": {},
"parentFolderId": {},
}
)
const (
// headerKeyPrefer is used to set query preferences
headerKeyPrefer = "Prefer"
// maxPageSizeHeaderFmt is used to indicate max page size
// preferences
maxPageSizeHeaderFmt = "odata.maxpagesize=%d"
// deltaMaxPageSize is the max page size to use for delta queries
deltaMaxPageSize = 200
idTypeFmt = "IdType=%q"
immutableIDType = "ImmutableId"
)
// -----------------------------------------------------------------------
// exchange.Query Option Section
// These functions can be used to filter a response on M365
// Graph queries and reduce / filter the amount of data returned
// which reduces the overall latency of complex calls
// -----------------------------------------------------------------------
// optionsForCalendars places allowed options for exchange.Calendar object
// @param moreOps should reflect elements from fieldsForCalendars
// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler
func optionsForCalendars(moreOps []string) (
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForCalendars)
if err != nil {
return nil, err
}
// should be a CalendarsRequestBuilderGetRequestConfiguration
requestParams := &users.ItemCalendarsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParams,
}
return options, nil
}
// optionsForCalendarsByID places allowed options for exchange.Calendar object
// @param moreOps should reflect elements from fieldsForCalendars
// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler
func optionsForCalendarsByID(moreOps []string) (
*users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForCalendars)
if err != nil {
return nil, err
}
// should be a CalendarsRequestBuilderGetRequestConfiguration
requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParams,
}
return options, nil
}
func optionsForContactFolderByID(moreOps []string) (
*users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForMailFoldersItem transforms the options into a more dynamic call for MailFoldersById.
// moreOps is a []string of options(e.g. "displayName", "isHidden")
// Returns first call in MailFoldersById().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForMailFoldersItem(
moreOps []string,
) (*users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForContactChildFolders builds a contacts child folders request.
func optionsForContactChildFolders(
moreOps []string,
) (*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// buildOptions - Utility Method for verifying if select options are valid for the m365 object type
// @return is a pair. The first is a string literal of allowable options based on the object type,
// the second is an error. An error is returned if an unsupported option or optionIdentifier was used
func buildOptions(fields []string, allowed map[string]struct{}) ([]string, error) {
returnedOptions := []string{"id"}
for _, entry := range fields {
_, ok := allowed[entry]
if !ok {
return nil, clues.New("unsupported field: " + entry)
}
}
return append(returnedOptions, fields...), nil
}
// buildPreferHeaders returns the headers we add to item delta page
// requests.
func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders {
var allHeaders []string
if pageSize {
allHeaders = append(allHeaders, fmt.Sprintf(maxPageSizeHeaderFmt, deltaMaxPageSize))
}
if immutableID {
allHeaders = append(allHeaders, fmt.Sprintf(idTypeFmt, immutableIDType))
}
headers := abstractions.NewRequestHeaders()
headers.Add(headerKeyPrefer, strings.Join(allHeaders, ","))
return headers
}

26
src/pkg/services/m365/api/query_params.go Normal file
View File

@ -0,0 +1,26 @@
package api
import (
"fmt"
"strings"
abstractions "github.com/microsoft/kiota-abstractions-go"
)
// buildPreferHeaders returns the headers we add to item delta page requests.
func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders {
var allHeaders []string
if pageSize {
allHeaders = append(allHeaders, fmt.Sprintf("odata.maxpagesize=%d", maxPageSize))
}
if immutableID {
allHeaders = append(allHeaders, `IdType="ImmutableId"`)
}
headers := abstractions.NewRequestHeaders()
headers.Add("Prefer", strings.Join(allHeaders, ","))
return headers
}

106
src/pkg/services/m365/api/users.go
View File

@ -14,6 +14,8 @@ import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/common/tform"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -394,90 +396,90 @@ func (c Users) getMailboxSettings(
additionalData := settings.GetAdditionalData()
mi.ArchiveFolder, err = toString(ctx, "archiveFolder", additionalData)
mi.ArchiveFolder, err = str.FromMapToAny("archiveFolder", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Timezone, err = toString(ctx, "timeZone", additionalData)
mi.Timezone, err = str.FromMapToAny("timeZone", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DateFormat, err = toString(ctx, "dateFormat", additionalData)
mi.DateFormat, err = str.FromMapToAny("dateFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.TimeFormat, err = toString(ctx, "timeFormat", additionalData)
mi.TimeFormat, err = str.FromMapToAny("timeFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Purpose, err = toString(ctx, "userPurpose", additionalData)
mi.Purpose, err = str.FromMapToAny("userPurpose", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DelegateMeetMsgDeliveryOpt, err = toString(ctx, "delegateMeetingMessageDeliveryOptions", additionalData)
mi.DelegateMeetMsgDeliveryOpt, err = str.FromMapToAny("delegateMeetingMessageDeliveryOptions", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode automatic replies settings
replySetting, err := toT[map[string]any](ctx, "automaticRepliesSetting", additionalData)
replySetting, err := tform.FromMapToAny[map[string]any]("automaticRepliesSetting", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.Status, err = toString(ctx, "status", replySetting)
mi.AutomaticRepliesSetting.Status, err = str.FromMapToAny("status", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalAudience, err = toString(ctx, "externalAudience", replySetting)
mi.AutomaticRepliesSetting.ExternalAudience, err = str.FromMapToAny("externalAudience", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalReplyMessage, err = toString(ctx, "externalReplyMessage", replySetting)
mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.FromMapToAny("externalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.InternalReplyMessage, err = toString(ctx, "internalReplyMessage", replySetting)
mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.FromMapToAny("internalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode scheduledStartDateTime
startDateTime, err := toT[map[string]any](ctx, "scheduledStartDateTime", replySetting)
startDateTime, err := tform.FromMapToAny[map[string]any]("scheduledStartDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = toString(ctx, "dateTime", startDateTime)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.FromMapToAny("dateTime", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = toString(ctx, "timeZone", startDateTime)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.FromMapToAny("timeZone", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
endDateTime, err := toT[map[string]any](ctx, "scheduledEndDateTime", replySetting)
endDateTime, err := tform.FromMapToAny[map[string]any]("scheduledEndDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = toString(ctx, "dateTime", endDateTime)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.FromMapToAny("dateTime", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = toString(ctx, "timeZone", endDateTime)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.FromMapToAny("timeZone", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// Language decode
language, err := toT[map[string]any](ctx, "language", additionalData)
language, err := tform.FromMapToAny[map[string]any]("language", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.DisplayName, err = toString(ctx, "displayName", language)
mi.Language.DisplayName, err = str.FromMapToAny("displayName", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.Locale, err = toString(ctx, "locale", language)
mi.Language.Locale, err = str.FromMapToAny("locale", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// working hours
workingHours, err := toT[map[string]any](ctx, "workingHours", additionalData)
workingHours, err := tform.FromMapToAny[map[string]any]("workingHours", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.StartTime, err = toString(ctx, "startTime", workingHours)
mi.WorkingHours.StartTime, err = str.FromMapToAny("startTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.EndTime, err = toString(ctx, "endTime", workingHours)
mi.WorkingHours.EndTime, err = str.FromMapToAny("endTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
timeZone, err := toT[map[string]any](ctx, "timeZone", workingHours)
timeZone, err := tform.FromMapToAny[map[string]any]("timeZone", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.TimeZone.Name, err = toString(ctx, "name", timeZone)
mi.WorkingHours.TimeZone.Name, err = str.FromMapToAny("name", timeZone)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
days, err := toT[[]any](ctx, "daysOfWeek", workingHours)
days, err := tform.FromMapToAny[[]any]("daysOfWeek", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
for _, day := range days {
s, err := anyToString(ctx, "dayOfTheWeek", day)
s, err := str.FromAny(day)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.DaysOfWeek = append(mi.WorkingHours.DaysOfWeek, s)
}
@ -510,53 +512,3 @@ func validateUser(item models.Userable) error {
return nil
}
func toString(ctx context.Context, key string, data map[string]any) (string, error) {
ctx = clues.Add(ctx, "setting_name", key)
if len(data) == 0 {
logger.Ctx(ctx).Info("not found: ", key)
return "", ErrMailBoxSettingsNotFound
}
return anyToString(ctx, key, data[key])
}
func anyToString(ctx context.Context, key string, val any) (string, error) {
if val == nil {
logger.Ctx(ctx).Info("nil value: ", key)
return "", ErrMailBoxSettingsNotFound
}
sp, ok := val.(*string)
if !ok {
logger.Ctx(ctx).Info("value is not a *string: ", key)
return "", ErrMailBoxSettingsNotFound
}
return ptr.Val(sp), nil
}
func toT[T any](ctx context.Context, key string, data map[string]any) (T, error) {
ctx = clues.Add(ctx, "setting_name", key)
if len(data) == 0 {
logger.Ctx(ctx).Info("not found: ", key)
return *new(T), ErrMailBoxSettingsNotFound
}
val := data[key]
if data == nil {
logger.Ctx(ctx).Info("nil value: ", key)
return *new(T), ErrMailBoxSettingsNotFound
}
value, ok := val.(T)
if !ok {
logger.Ctx(ctx).Info(fmt.Sprintf("unexpected type for %s: %T", key, val))
return *new(T), ErrMailBoxSettingsNotFound
}
return value, nil
}

5
src/pkg/storage/s3.go
View File

@ -6,6 +6,7 @@ import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
)
type S3Config struct {
@ -68,8 +69,8 @@ func (s Storage) S3Config() (S3Config, error) {
c.Bucket = orEmptyString(s.Config[keyS3Bucket])
c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
c.Prefix = orEmptyString(s.Config[keyS3Prefix])
c.DoNotUseTLS = common.ParseBool(s.Config[keyS3DoNotUseTLS])
c.DoNotVerifyTLS = common.ParseBool(s.Config[keyS3DoNotVerifyTLS])
c.DoNotUseTLS = str.ParseBool(s.Config[keyS3DoNotUseTLS])
c.DoNotVerifyTLS = str.ParseBool(s.Config[keyS3DoNotVerifyTLS])
}
return c, c.validate()