Merge branch 'main' into beta-library-switch

This commit is contained in:
Danny Adams 2023-01-20 09:40:57 -05:00
commit 2f125d44a6
16 changed files with 210 additions and 77 deletions

View File

@ -13,8 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Beta Libraries are included in package. This can lead to long build times.
- msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times.
- Handle case where user's drive has not been initialized
- Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163))
## [v0.1.0] (alpha) - 2023-01-13

View File

@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
github.com/aws/aws-sdk-go v1.44.181
github.com/aws/aws-sdk-go v1.44.183
github.com/aws/aws-xray-sdk-go v1.8.0
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1

View File

@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go v1.44.181 h1:w4OzE8bwIVo62gUTAp/uEFO2HSsUtf1pjXpSs36cluY=
github.com/aws/aws-sdk-go v1.44.181/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk=
github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY=
github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

View File

@ -58,6 +58,28 @@ func (c Events) DeleteCalendar(
return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
}
func (c Events) GetContainerByID(
ctx context.Context,
userID, containerID string,
) (graph.Container, error) {
service, err := c.service()
if err != nil {
return nil, err
}
ofc, err := optionsForCalendarsByID([]string{"name", "owner"})
if err != nil {
return nil, errors.Wrap(err, "options for event calendar")
}
cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc)
if err != nil {
return nil, err
}
return graph.CalendarDisplayable{Calendarable: cal}, nil
}
// GetItem retrieves an Eventable item.
func (c Events) GetItem(
ctx context.Context,
@ -183,14 +205,9 @@ func (c Events) GetAddedAndRemovedItemIDs(
errs *multierror.Error
)
options, err := optionsForEventsByCalendarDelta([]string{"id"})
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
if len(oldDelta) > 0 {
builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr := &eventPager{service, builder, options}
pgr := &eventPager{service, builder, nil}
added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
// note: happy path, not the error condition
@ -217,7 +234,7 @@ func (c Events) GetAddedAndRemovedItemIDs(
// works as intended (until, at least, we want to _not_ call the beta anymore).
rawURL := fmt.Sprintf(eventBetaDeltaURLTemplate, user, calendarID)
builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(rawURL, service.Adapter())
pgr := &eventPager{service, builder, options}
pgr := &eventPager{service, builder, nil}
added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
if err != nil {
@ -251,7 +268,7 @@ func (c Events) Serialize(
defer writer.Close()
if *event.GetHasAttachments() {
if *event.GetHasAttachments() || support.HasAttachments(event.GetBody()) {
// getting all the attachments might take a couple attempts due to filesize
var retriesErr error

View File

@ -257,7 +257,7 @@ func (c Mail) Serialize(
defer writer.Close()
if *msg.GetHasAttachments() {
if *msg.GetHasAttachments() || support.HasAttachments(msg.GetBody()) {
// getting all the attachments might take a couple attempts due to filesize
var retriesErr error

View File

@ -21,18 +21,6 @@ var (
"owner": {},
}
fieldsForEvents = map[string]struct{}{
"calendar": {},
"end": {},
"id": {},
"isOnlineMeeting": {},
"isReminderOn": {},
"responseStatus": {},
"responseRequested": {},
"showAs": {},
"subject": {},
}
fieldsForFolders = map[string]struct{}{
"childFolderCount": {},
"displayName": {},
@ -112,6 +100,28 @@ func optionsForCalendars(moreOps []string) (
return options, nil
}
// optionsForCalendarsByID places allowed options for exchange.Calendar object
// @param moreOps should reflect elements from fieldsForCalendars
// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler
func optionsForCalendarsByID(moreOps []string) (
*users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForCalendars)
if err != nil {
return nil, err
}
// should be a CalendarsRequestBuilderGetRequestConfiguration
requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParams,
}
return options, nil
}
// optionsForContactFolders places allowed options for exchange.ContactFolder object
// @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler
func optionsForContactFolders(moreOps []string) (
@ -213,26 +223,6 @@ func optionsForContactFoldersItemDelta(
return options, nil
}
// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar
func optionsForEventsByCalendarDelta(
moreOps []string,
) (*users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForEvents)
if err != nil {
return nil, err
}
requestParameters := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForContactChildFolders builds a contacts child folders request.
func optionsForContactChildFolders(
moreOps []string,

View File

@ -6,6 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
@ -14,9 +15,43 @@ var _ graph.ContainerResolver = &eventCalendarCache{}
type eventCalendarCache struct {
*containerResolver
enumer containersEnumerator
getter containerGetter
userID string
}
// init ensures that the structure's fields are initialized.
// Fields Initialized when cache == nil:
// [mc.cache]
func (ecc *eventCalendarCache) init(
ctx context.Context,
) error {
if ecc.containerResolver == nil {
ecc.containerResolver = newContainerResolver()
}
return ecc.populateEventRoot(ctx)
}
// populateEventRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+
// DefaultCalendar is the traditional "Calendar".
// Action ensures that cache will stop at appropriate level.
// @error iff the struct is not properly instantiated
func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error {
container := DefaultCalendar
f, err := ecc.getter.GetContainerByID(ctx, ecc.userID, container)
if err != nil {
return errors.Wrap(err, "fetching calendar "+support.ConnectorStackErrorTrace(err))
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(container))
if err := ecc.addFolder(temp); err != nil {
return errors.Wrap(err, "initializing calendar resolver")
}
return nil
}
// Populate utility function for populating eventCalendarCache.
// Executes 1 additional Graph Query
// @param baseID: ignored. Present to conform to interface
@ -25,8 +60,8 @@ func (ecc *eventCalendarCache) Populate(
baseID string,
baseContainerPath ...string,
) error {
if ecc.containerResolver == nil {
ecc.containerResolver = newContainerResolver()
if err := ecc.init(ctx); err != nil {
return errors.Wrap(err, "initializing")
}
err := ecc.enumer.EnumerateContainers(ctx, ecc.userID, "", ecc.addFolder)
@ -34,6 +69,10 @@ func (ecc *eventCalendarCache) Populate(
return errors.Wrap(err, "enumerating containers")
}
if err := ecc.populatePaths(ctx); err != nil {
return errors.Wrap(err, "establishing calendar paths")
}
return nil
}

View File

@ -13,6 +13,7 @@ import (
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
@ -197,10 +198,6 @@ func (col *Collection) streamItems(ctx context.Context) {
semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize)
defer close(semaphoreCh)
errUpdater := func(user string, err error) {
errs = support.WrapAndAppend(user, err, errs)
}
// delete all removed items
for id := range col.removed {
semaphoreCh <- struct{}{}
@ -226,6 +223,14 @@ func (col *Collection) streamItems(ctx context.Context) {
}(id)
}
updaterMu := sync.Mutex{}
errUpdater := func(user string, err error) {
updaterMu.Lock()
defer updaterMu.Unlock()
errs = support.WrapAndAppend(user, err, errs)
}
// add any new items
for id := range col.added {
if col.ctrl.FailFast && errs != nil {
@ -258,7 +263,18 @@ func (col *Collection) streamItems(ctx context.Context) {
}
if err != nil {
errUpdater(user, err)
// Don't report errors for deleted items as there's no way for us to
// back up data that is gone. Chalk them up as a "success" though since
// there's really nothing we can do and not reporting it will make the
// status code upset cause we won't have the same number of results as
// attempted items.
if e := graph.IsErrDeletedInFlight(err); e != nil {
atomic.AddInt64(&success, 1)
return
}
errUpdater(user, support.ConnectorStackErrorTraceWrap(err, "fetching item"))
return
}

View File

@ -51,6 +51,7 @@ func (suite *CacheResolverSuite) TestPopulate() {
return &eventCalendarCache{
userID: tester.M365UserID(t),
enumer: ac.Events(),
getter: ac.Events(),
}
}

View File

@ -22,14 +22,25 @@ type mailFolderCache struct {
userID string
}
// init ensures that the structure's fields are initialized.
// Fields Initialized when cache == nil:
// [mc.cache]
func (mc *mailFolderCache) init(
ctx context.Context,
) error {
if mc.containerResolver == nil {
mc.containerResolver = newContainerResolver()
}
return mc.populateMailRoot(ctx)
}
// populateMailRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+
// rootFolderAlias is the top-level directory for exchange.Mail.
// DefaultMailFolder is the traditional "Inbox" for exchange.Mail
// Action ensures that cache will stop at appropriate level.
// @error iff the struct is not properly instantiated
func (mc *mailFolderCache) populateMailRoot(
ctx context.Context,
) error {
func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error {
for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} {
var directory string
@ -76,16 +87,3 @@ func (mc *mailFolderCache) Populate(
return nil
}
// init ensures that the structure's fields are initialized.
// Fields Initialized when cache == nil:
// [mc.cache]
func (mc *mailFolderCache) init(
ctx context.Context,
) error {
if mc.containerResolver == nil {
mc.containerResolver = newContainerResolver()
}
return mc.populateMailRoot(ctx)
}

View File

@ -66,9 +66,11 @@ func PopulateExchangeContainerResolver(
cacheRoot = DefaultContactFolder
case path.EventsCategory:
ecc := ac.Events()
res = &eventCalendarCache{
userID: qp.ResourceOwner,
enumer: ac.Events(),
getter: ecc,
enumer: ecc,
}
cacheRoot = DefaultCalendar

View File

@ -507,9 +507,11 @@ func CreateContainerDestinaion(
case path.EventsCategory:
if directoryCache == nil {
ace := ac.Events()
ecc := &eventCalendarCache{
userID: user,
enumer: ac.Events(),
getter: ace,
enumer: ace,
}
caches[category] = ecc
newCache = true

View File

@ -76,7 +76,8 @@ const (
itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children"
itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s"
itemNotFoundErrorCode = "itemNotFound"
userDoesNotHaveDrive = "BadRequest Unable to retrieve user's mysite URL"
userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL"
userMysiteNotFound = "ResourceNotFound User's mysite not found"
)
// Enumerates the drives for the specified user
@ -134,7 +135,8 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod
r, err = service.Client().UsersById(user).Drives().Get(ctx, nil)
if err != nil {
detailedError := support.ConnectorStackErrorTrace(err)
if strings.Contains(detailedError, userDoesNotHaveDrive) {
if strings.Contains(detailedError, userMysiteURLNotFound) ||
strings.Contains(detailedError, userMysiteNotFound) {
logger.Ctx(ctx).Debugf("User %s does not have a drive", user)
return make([]models.Driveable, 0), nil // no license
}

View File

@ -1,6 +1,8 @@
package support
import (
"strings"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
js "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-beta-sdk-go/models"
@ -71,3 +73,14 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) {
return list, nil
}
func HasAttachments(body models.ItemBodyable) bool {
if body.GetContent() == nil || body.GetContentType() == nil ||
*body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 {
return false
}
content := *body.GetContent()
return strings.Contains(content, "src=\"cid:")
}

View File

@ -3,6 +3,7 @@ package support
import (
"testing"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -159,3 +160,56 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() {
})
}
}
func (suite *DataSupportSuite) TestHasAttachments() {
tests := []struct {
name string
hasAttachment assert.BoolAssertionFunc
getBodyable func(t *testing.T) models.ItemBodyable
}{
{
name: "Mock w/out attachment",
hasAttachment: assert.False,
getBodyable: func(t *testing.T) models.ItemBodyable {
byteArray := mockconnector.GetMockMessageWithBodyBytes(
"Test",
"This is testing",
"This is testing",
)
message, err := CreateMessageFromBytes(byteArray)
require.NoError(t, err)
return message.GetBody()
},
},
{
name: "Mock w/ inline attachment",
hasAttachment: assert.True,
getBodyable: func(t *testing.T) models.ItemBodyable {
byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy")
message, err := CreateMessageFromBytes(byteArray)
require.NoError(t, err)
return message.GetBody()
},
},
{
name: "Edge Case",
hasAttachment: assert.True,
getBodyable: func(t *testing.T) models.ItemBodyable {
//nolint:lll
content := "<html><head>\r\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"><style type=\"text/css\" style=\"display:none\">\r\n<!--\r\np\r\n\t{margin-top:0;\r\n\tmargin-bottom:0}\r\n-->\r\n</style></head><body dir=\"ltr\"><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Happy New Year,</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture?&nbsp;</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><img class=\"FluidPluginCopy ContentPasted0 w-2070 h-1380\" size=\"5854817\" data-outlook-trace=\"F:1|T:1\" src=\"cid:85f4faa3-9851-40c7-ba0a-e63dce1185f9\" style=\"max-width:100%\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Let me know if this meets our culture requirements.</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Warm Regards,</div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\"><br></div><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0); background-color:rgb(255,255,255)\">Dustin</div></body></html>"
body := models.NewItemBody()
body.SetContent(&content)
cat := models.HTML_BODYTYPE
body.SetContentType(&cat)
return body
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
found := HasAttachments(test.getBodyable(t))
test.hasAttachment(t, found)
})
}
}

View File

@ -111,8 +111,7 @@ func readTestConfig() (map[string]string, error) {
TestCfgUserID,
os.Getenv(EnvCorsoM365TestUserID),
vpr.GetString(TestCfgUserID),
"lynner@8qzvrj.onmicrosoft.com",
//"lidiah@8qzvrj.onmicrosoft.com",
"conneri@8qzvrj.onmicrosoft.com",
)
fallbackTo(
testEnv,
@ -120,7 +119,6 @@ func readTestConfig() (map[string]string, error) {
os.Getenv(EnvCorsoSecondaryM365TestUserID),
vpr.GetString(TestCfgSecondaryUserID),
"lidiah@8qzvrj.onmicrosoft.com",
//"lynner@8qzvrj.onmicrosoft.com",
)
fallbackTo(
testEnv,
@ -134,7 +132,7 @@ func readTestConfig() (map[string]string, error) {
TestCfgLoadTestOrgUsers,
os.Getenv(EnvCorsoM365LoadTestOrgUsers),
vpr.GetString(TestCfgLoadTestOrgUsers),
"lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com",
"lidiah@8qzvrj.onmicrosoft.com,conneri@8qzvrj.onmicrosoft.com",
)
fallbackTo(
testEnv,