larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get OneDrive item with getM365 (#2791)

Browse Source 
Add support for onedrive in getM365

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2023-04-06 11:02:03 +05:30 committed by GitHub
parent bebd83474a
commit baddd9fc83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 362 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Makefile
View File

@ -79,4 +79,7 @@ load-test:
-mutexprofile=mutex.prof \
-trace=trace.out \
-outputdir=test_results \
./pkg/repository/loadtest/repository_load_test.go
./pkg/repository/loadtest/repository_load_test.go
getM365:
go build -o getM365 cmd/getM365/main.go

27
src/cli/print/print.go
View File

@ -62,7 +62,7 @@ func StderrWriter(ctx context.Context) io.Writer {
}
// ---------------------------------------------------------------------------------------------------------
// Helper funcs
// Exported interface
// ---------------------------------------------------------------------------------------------------------
// Only tells the CLI to only display this error, preventing the usage
@ -110,6 +110,15 @@ func Infof(ctx context.Context, t string, s ...any) {
outf(getRootCmd(ctx).ErrOrStderr(), t, s...)
}
// PrettyJSON prettifies and prints the value.
func PrettyJSON(ctx context.Context, p minimumPrintabler) {
if p == nil {
Err(ctx, "<nil>")
}
outputJSON(getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
}
// out is the testable core of exported print funcs
func out(w io.Writer, s ...any) {
if len(s) == 0 {
@ -135,8 +144,7 @@ func outf(w io.Writer, t string, s ...any) {
// ---------------------------------------------------------------------------------------------------------
type Printable interface {
// reduces the struct to a minimized format for easier human consumption
MinimumPrintable() any
minimumPrintabler
// should list the property names of the values surfaced in Values()
Headers() []string
// list of values for tabular or csv formatting
@ -145,6 +153,11 @@ type Printable interface {
Values() []string
}
type minimumPrintabler interface {
// reduces the struct to a minimized format for easier human consumption
MinimumPrintable() any
}
// Item prints the printable, according to the caller's requested format.
func Item(ctx context.Context, p Printable) {
printItem(getRootCmd(ctx).OutOrStdout(), p)
@ -216,13 +229,17 @@ func outputTable(w io.Writer, ps []Printable) {
// JSON
// ------------------------------------------------------------------------------------------
func outputJSON(w io.Writer, p Printable, debug bool) {
func outputJSON(w io.Writer, p minimumPrintabler, debug bool) {
if debug {
printJSON(w, p)
return
}
printJSON(w, p.MinimumPrintable())
if debug {
printJSON(w, p)
} else {
printJSON(w, p.MinimumPrintable())
}
}
func outputJSONArr(w io.Writer, ps []Printable, debug bool) {

112
src/cmd/getM365/getItem.go → src/cmd/getM365/exchange/get_item.go
View File

@ -1,8 +1,8 @@
// getItem.go is a source file designed to retrieve an m365 object from an
// get_item.go is a source file designed to retrieve an m365 object from an
// existing M365 account. Data displayed is representative of the current
// serialization abstraction versioning used by Microsoft Graph and stored by Corso.
package main
package exchange
import (
"context"
@ -14,76 +14,65 @@ import (
kw "github.com/microsoft/kiota-serialization-json-go"
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
var getCmd = &cobra.Command{
Use: "get",
Short: "Get a M365ID item JSON",
RunE: handleGetCommand,
}
// Required inputs from user for command execution
var (
tenant, user, m365ID, category string
user, tenant, m365ID, category string
)
// main function will produce the JSON String for a given m365 object of a
// user. Displayed Objects can be used as inputs for Mockable data
// Supports:
// - exchange (contacts, email, and events)
// Input: go run ./getItem.go --user <user>
//
// --m365ID <m365ID> --category <oneof: contacts, email, events>
func main() {
ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
ctx = SetRootCmd(ctx, getCmd)
defer logger.Flush(ctx)
fs := getCmd.PersistentFlags()
fs.StringVar(&user, "user", "", "m365 user id of M365 user")
fs.StringVar(&tenant, "tenant", "",
"m365 Tenant: m365 identifier for the tenant, not required if active in OS Environment")
fs.StringVar(&m365ID, "m365ID", "", "m365 identifier for object to be created")
fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events or files)") // files not supported
cobra.CheckErr(getCmd.MarkPersistentFlagRequired("user"))
cobra.CheckErr(getCmd.MarkPersistentFlagRequired("m365ID"))
cobra.CheckErr(getCmd.MarkPersistentFlagRequired("category"))
if err := getCmd.ExecuteContext(ctx); err != nil {
logger.Flush(ctx)
os.Exit(1)
func AddCommands(parent *cobra.Command) {
exCmd := &cobra.Command{
Use: "exchange",
Short: "Get an M365ID item JSON",
RunE: handleExchangeCmd,
}
fs := exCmd.PersistentFlags()
fs.StringVar(&m365ID, "id", "", "m365 identifier for object")
fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events)")
fs.StringVar(&user, "user", "", "m365 user id of M365 user")
fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant")
cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user"))
cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id"))
cobra.CheckErr(exCmd.MarkPersistentFlagRequired("category"))
parent.AddCommand(exCmd)
}
func handleGetCommand(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
func handleExchangeCmd(cmd *cobra.Command, args []string) error {
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
_, creds, err := getGC(ctx)
if err != nil {
return err
tid := common.First(tenant, os.Getenv(account.AzureTenantID))
ctx := clues.Add(
cmd.Context(),
"item_id", m365ID,
"resource_owner", user,
"tenant", tid)
creds := account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: tid,
}
err = runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true))
err := runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true))
if err != nil {
return Only(ctx, clues.Wrap(err, "Error displaying item: "+m365ID))
cmd.SilenceUsage = true
cmd.SilenceErrors = true
return clues.Wrap(err, "getting item")
}
return nil
@ -165,30 +154,3 @@ func getItem(
return itm.Serialize(ctx, sp, user, itemID)
}
//-------------------------------------------------------------------------------
// Helpers
//-------------------------------------------------------------------------------
func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config, error) {
// get account info
m365Cfg := account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)),
}
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil {
return nil, m365Cfg, Only(ctx, clues.Wrap(err, "finding m365 account details"))
}
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs)
if err != nil {
return nil, m365Cfg, Only(ctx, clues.Wrap(err, "connecting to graph API"))
}
return gc, m365Cfg, nil
}

32
src/cmd/getM365/main.go Normal file
View File

@ -0,0 +1,32 @@
package main
import (
"context"
"os"
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cmd/getM365/exchange"
"github.com/alcionai/corso/src/cmd/getM365/onedrive"
"github.com/alcionai/corso/src/pkg/logger"
)
var rootCmd = &cobra.Command{
Use: "getM365",
}
func main() {
ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
ctx = SetRootCmd(ctx, rootCmd)
defer logger.Flush(ctx)
exchange.AddCommands(rootCmd)
onedrive.AddCommands(rootCmd)
if err := rootCmd.Execute(); err != nil {
Err(ctx, err)
os.Exit(1)
}
}

207
src/cmd/getM365/onedrive/get_item.go Normal file
View File

@ -0,0 +1,207 @@
// get_item.go is a source file designed to retrieve an m365 object from an
// existing M365 account. Data displayed is representative of the current
// serialization abstraction versioning used by Microsoft Graph and stored by Corso.
package onedrive
import (
"context"
"encoding/json"
"io"
"net/http"
"os"
"github.com/alcionai/clues"
"github.com/microsoft/kiota-abstractions-go/serialization"
kjson "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
)
const downloadURLKey = "@microsoft.graph.downloadUrl"
// Required inputs from user for command execution
var (
user, tenant, m365ID string
)
func AddCommands(parent *cobra.Command) {
exCmd := &cobra.Command{
Use: "onedrive",
Short: "Get an M365ID item",
RunE: handleOneDriveCmd,
}
fs := exCmd.PersistentFlags()
fs.StringVar(&m365ID, "id", "", "m365 identifier for object")
fs.StringVar(&user, "user", "", "m365 user id of M365 user")
fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant")
cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user"))
cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id"))
parent.AddCommand(exCmd)
}
func handleOneDriveCmd(cmd *cobra.Command, args []string) error {
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
tid := common.First(tenant, os.Getenv(account.AzureTenantID))
ctx := clues.Add(
cmd.Context(),
"item_id", m365ID,
"resource_owner", user,
"tenant", tid)
// get account info
creds := account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: tid,
}
// todo: swap to drive api client, when finished.
adpt, err := graph.CreateAdapter(tid, creds.AzureClientID, creds.AzureClientSecret)
if err != nil {
return Only(ctx, clues.Wrap(err, "creating graph adapter"))
}
err = runDisplayM365JSON(ctx, graph.NewService(adpt), creds, user, m365ID)
if err != nil {
cmd.SilenceUsage = true
cmd.SilenceErrors = true
return Only(ctx, clues.Wrap(err, "getting item"))
}
return nil
}
type itemData struct {
Size int `json:"size"`
}
type itemPrintable struct {
Info json.RawMessage `json:"info"`
Permissions json.RawMessage `json:"permissions"`
Data itemData `json:"data"`
}
func (i itemPrintable) MinimumPrintable() any {
return i
}
func runDisplayM365JSON(
ctx context.Context,
srv graph.Servicer,
creds account.M365Config,
user, itemID string,
) error {
drive, err := api.GetDriveByID(ctx, srv, user)
if err != nil {
return err
}
driveID := ptr.Val(drive.GetId())
it := itemPrintable{}
item, err := api.GetDriveItem(ctx, srv, driveID, itemID)
if err != nil {
return err
}
if item != nil {
content, err := getDriveItemContent(item)
if err != nil {
return err
}
// We could get size from item.GetSize(), but the
// getDriveItemContent call is to ensure that we are able to
// download the file.
it.Data.Size = len(content)
}
sInfo, err := serializeObject(item)
if err != nil {
return err
}
err = json.Unmarshal([]byte(sInfo), &it.Info)
if err != nil {
return err
}
perms, err := api.GetItemPermission(ctx, srv, driveID, itemID)
if err != nil {
return err
}
sPerms, err := serializeObject(perms)
if err != nil {
return err
}
err = json.Unmarshal([]byte(sPerms), &it.Permissions)
if err != nil {
return err
}
PrettyJSON(ctx, it)
return nil
}
func serializeObject(data serialization.Parsable) (string, error) {
sw := kjson.NewJsonSerializationWriter()
err := sw.WriteObjectValue("", data)
if err != nil {
return "", clues.Wrap(err, "writing serializing info")
}
content, err := sw.GetSerializedContent()
if err != nil {
return "", clues.Wrap(err, "getting serializing info")
}
return string(content), err
}
func getDriveItemContent(item models.DriveItemable) ([]byte, error) {
url, ok := item.GetAdditionalData()[downloadURLKey].(*string)
if !ok {
return nil, clues.New("get download url")
}
req, err := http.NewRequest(http.MethodGet, *url, nil)
if err != nil {
return nil, clues.New("create download request").With("error", err)
}
hc := graph.HTTPClient(graph.NoTimeout())
resp, err := hc.Do(req)
if err != nil {
return nil, clues.New("download item").With("error", err)
}
content, err := io.ReadAll(resp.Body)
if err != nil {
return nil, clues.New("read downloaded item").With("error", err)
}
return content, nil
}

3
src/internal/connector/graph/service.go
View File

@ -29,6 +29,7 @@ import (
const (
logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS"
log2xxGraphRequestsEnvKey = "LOG_2XX_GRAPH_REQUESTS"
log2xxGraphResponseEnvKey = "LOG_2XX_GRAPH_RESPONSES"
retryAttemptHeader = "Retry-Attempt"
retryAfterHeader = "Retry-After"
defaultMaxRetries = 3
@ -368,7 +369,7 @@ func (handler *LoggingMiddleware) Intercept(
// If api logging is toggled, log a body-less dump of the request/resp.
if (resp.StatusCode / 100) == 2 {
if logger.DebugAPI || os.Getenv(log2xxGraphRequestsEnvKey) != "" {
log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, false))
log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != ""))
}
return resp, err

56
src/internal/connector/onedrive/api/drive.go
View File

@ -203,10 +203,6 @@ func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error)
return getValues[models.Driveable](l)
}
// ---------------------------------------------------------------------------
// Drive Paging
// ---------------------------------------------------------------------------
// DrivePager pages through different types of drive owners
type DrivePager interface {
GetPage(context.Context) (api.PageLinker, error)
@ -275,3 +271,55 @@ func GetAllDrives(
return ds, nil
}
// generic drive item getter
func GetDriveItem(
ctx context.Context,
srv graph.Servicer,
driveID, itemID string,
) (models.DriveItemable, error) {
di, err := srv.Client().
DrivesById(driveID).
ItemsById(itemID).
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting item")
}
return di, nil
}
func GetItemPermission(
ctx context.Context,
service graph.Servicer,
driveID, itemID string,
) (models.PermissionCollectionResponseable, error) {
perm, err := service.
Client().
DrivesById(driveID).
ItemsById(itemID).
Permissions().
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting item metadata").With("item_id", itemID)
}
return perm, nil
}
func GetDriveByID(
ctx context.Context,
srv graph.Servicer,
userID string,
) (models.Driveable, error) {
//revive:enable:context-as-argument
d, err := srv.Client().
UsersById(userID).
Drive().
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting drive")
}
return d, nil
}

5
src/internal/connector/onedrive/collection.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
@ -166,11 +167,11 @@ func NewCollection(
// Allows tests to set a mock populator
switch source {
case SharePointSource:
c.itemGetter = getDriveItem
c.itemGetter = api.GetDriveItem
c.itemReader = sharePointItemReader
c.itemMetaReader = sharePointItemMetaReader
default:
c.itemGetter = getDriveItem
c.itemGetter = api.GetDriveItem
c.itemReader = oneDriveItemReader
c.itemMetaReader = oneDriveItemMetaReader
}

24
src/internal/connector/onedrive/item.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/uploadsession"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
@ -26,20 +27,6 @@ const (
downloadURLKey = "@microsoft.graph.downloadUrl"
)
// generic drive item getter
func getDriveItem(
ctx context.Context,
srv graph.Servicer,
driveID, itemID string,
) (models.DriveItemable, error) {
di, err := srv.Client().DrivesById(driveID).ItemsById(itemID).Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting item")
}
return di, nil
}
// sharePointItemReader will return a io.ReadCloser for the specified item
// It crafts this by querying M365 for a download URL for the item
// and using a http client to initialize a reader
@ -229,14 +216,9 @@ func driveItemPermissionInfo(
driveID string,
itemID string,
) ([]UserPermission, error) {
perm, err := service.
Client().
DrivesById(driveID).
ItemsById(itemID).
Permissions().
Get(ctx, nil)
perm, err := api.GetItemPermission(ctx, service, driveID, itemID)
if err != nil {
return nil, graph.Wrap(ctx, err, "fetching item permissions").With("item_id", itemID)
return nil, err
}
uperms := filterUserPermissions(ctx, perm.GetValue())