From baddd9fc83e3192b7ad73ae18e480f3409e2f096 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 6 Apr 2023 11:02:03 +0530 Subject: [PATCH] Get OneDrive item with getM365 (#2791) Add support for onedrive in getM365 --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/Makefile | 5 +- src/cli/print/print.go | 27 ++- .../{getItem.go => exchange/get_item.go} | 112 ++++------ src/cmd/getM365/main.go | 32 +++ src/cmd/getM365/onedrive/get_item.go | 207 ++++++++++++++++++ src/internal/connector/graph/service.go | 3 +- src/internal/connector/onedrive/api/drive.go | 56 ++++- src/internal/connector/onedrive/collection.go | 5 +- src/internal/connector/onedrive/item.go | 24 +- 9 files changed, 362 insertions(+), 109 deletions(-) rename src/cmd/getM365/{getItem.go => exchange/get_item.go} (50%) create mode 100644 src/cmd/getM365/main.go create mode 100644 src/cmd/getM365/onedrive/get_item.go diff --git a/src/Makefile b/src/Makefile index 6f75be16b..fff36d78c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -79,4 +79,7 @@ load-test: -mutexprofile=mutex.prof \ -trace=trace.out \ -outputdir=test_results \ - ./pkg/repository/loadtest/repository_load_test.go \ No newline at end of file + ./pkg/repository/loadtest/repository_load_test.go + +getM365: + go build -o getM365 cmd/getM365/main.go \ No newline at end of file diff --git a/src/cli/print/print.go b/src/cli/print/print.go index c15643b83..5ab61acca 100644 --- a/src/cli/print/print.go +++ b/src/cli/print/print.go @@ -62,7 +62,7 @@ func StderrWriter(ctx context.Context) io.Writer { } // --------------------------------------------------------------------------------------------------------- -// Helper funcs +// Exported interface // --------------------------------------------------------------------------------------------------------- // Only tells the CLI to only display this error, preventing the usage @@ -110,6 +110,15 @@ func Infof(ctx context.Context, t string, s ...any) { outf(getRootCmd(ctx).ErrOrStderr(), t, s...) } +// PrettyJSON prettifies and prints the value. +func PrettyJSON(ctx context.Context, p minimumPrintabler) { + if p == nil { + Err(ctx, "") + } + + outputJSON(getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug) +} + // out is the testable core of exported print funcs func out(w io.Writer, s ...any) { if len(s) == 0 { @@ -135,8 +144,7 @@ func outf(w io.Writer, t string, s ...any) { // --------------------------------------------------------------------------------------------------------- type Printable interface { - // reduces the struct to a minimized format for easier human consumption - MinimumPrintable() any + minimumPrintabler // should list the property names of the values surfaced in Values() Headers() []string // list of values for tabular or csv formatting @@ -145,6 +153,11 @@ type Printable interface { Values() []string } +type minimumPrintabler interface { + // reduces the struct to a minimized format for easier human consumption + MinimumPrintable() any +} + // Item prints the printable, according to the caller's requested format. func Item(ctx context.Context, p Printable) { printItem(getRootCmd(ctx).OutOrStdout(), p) @@ -216,13 +229,17 @@ func outputTable(w io.Writer, ps []Printable) { // JSON // ------------------------------------------------------------------------------------------ -func outputJSON(w io.Writer, p Printable, debug bool) { +func outputJSON(w io.Writer, p minimumPrintabler, debug bool) { if debug { printJSON(w, p) return } - printJSON(w, p.MinimumPrintable()) + if debug { + printJSON(w, p) + } else { + printJSON(w, p.MinimumPrintable()) + } } func outputJSONArr(w io.Writer, ps []Printable, debug bool) { diff --git a/src/cmd/getM365/getItem.go b/src/cmd/getM365/exchange/get_item.go similarity index 50% rename from src/cmd/getM365/getItem.go rename to src/cmd/getM365/exchange/get_item.go index 829c70b1a..6e7f9022a 100644 --- a/src/cmd/getM365/getItem.go +++ b/src/cmd/getM365/exchange/get_item.go @@ -1,8 +1,8 @@ -// getItem.go is a source file designed to retrieve an m365 object from an +// get_item.go is a source file designed to retrieve an m365 object from an // existing M365 account. Data displayed is representative of the current // serialization abstraction versioning used by Microsoft Graph and stored by Corso. -package main +package exchange import ( "context" @@ -14,76 +14,65 @@ import ( kw "github.com/microsoft/kiota-serialization-json-go" "github.com/spf13/cobra" - . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/exchange/api" - "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) -var getCmd = &cobra.Command{ - Use: "get", - Short: "Get a M365ID item JSON", - RunE: handleGetCommand, -} - // Required inputs from user for command execution var ( - tenant, user, m365ID, category string + user, tenant, m365ID, category string ) -// main function will produce the JSON String for a given m365 object of a -// user. Displayed Objects can be used as inputs for Mockable data -// Supports: -// - exchange (contacts, email, and events) -// Input: go run ./getItem.go --user -// -// --m365ID --category -func main() { - ctx, _ := logger.SeedLevel(context.Background(), logger.Development) - ctx = SetRootCmd(ctx, getCmd) - - defer logger.Flush(ctx) - - fs := getCmd.PersistentFlags() - fs.StringVar(&user, "user", "", "m365 user id of M365 user") - fs.StringVar(&tenant, "tenant", "", - "m365 Tenant: m365 identifier for the tenant, not required if active in OS Environment") - fs.StringVar(&m365ID, "m365ID", "", "m365 identifier for object to be created") - fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events or files)") // files not supported - - cobra.CheckErr(getCmd.MarkPersistentFlagRequired("user")) - cobra.CheckErr(getCmd.MarkPersistentFlagRequired("m365ID")) - cobra.CheckErr(getCmd.MarkPersistentFlagRequired("category")) - - if err := getCmd.ExecuteContext(ctx); err != nil { - logger.Flush(ctx) - os.Exit(1) +func AddCommands(parent *cobra.Command) { + exCmd := &cobra.Command{ + Use: "exchange", + Short: "Get an M365ID item JSON", + RunE: handleExchangeCmd, } + + fs := exCmd.PersistentFlags() + fs.StringVar(&m365ID, "id", "", "m365 identifier for object") + fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events)") + fs.StringVar(&user, "user", "", "m365 user id of M365 user") + fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") + + cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) + cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) + cobra.CheckErr(exCmd.MarkPersistentFlagRequired("category")) + + parent.AddCommand(exCmd) } -func handleGetCommand(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - +func handleExchangeCmd(cmd *cobra.Command, args []string) error { if utils.HasNoFlagsAndShownHelp(cmd) { return nil } - _, creds, err := getGC(ctx) - if err != nil { - return err + tid := common.First(tenant, os.Getenv(account.AzureTenantID)) + + ctx := clues.Add( + cmd.Context(), + "item_id", m365ID, + "resource_owner", user, + "tenant", tid) + + creds := account.M365Config{ + M365: credentials.GetM365(), + AzureTenantID: tid, } - err = runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true)) + err := runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true)) if err != nil { - return Only(ctx, clues.Wrap(err, "Error displaying item: "+m365ID)) + cmd.SilenceUsage = true + cmd.SilenceErrors = true + + return clues.Wrap(err, "getting item") } return nil @@ -165,30 +154,3 @@ func getItem( return itm.Serialize(ctx, sp, user, itemID) } - -//------------------------------------------------------------------------------- -// Helpers -//------------------------------------------------------------------------------- - -func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config, error) { - // get account info - m365Cfg := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)), - } - - acct, err := account.NewAccount(account.ProviderM365, m365Cfg) - if err != nil { - return nil, m365Cfg, Only(ctx, clues.Wrap(err, "finding m365 account details")) - } - - // TODO: log/print recoverable errors - errs := fault.New(false) - - gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs) - if err != nil { - return nil, m365Cfg, Only(ctx, clues.Wrap(err, "connecting to graph API")) - } - - return gc, m365Cfg, nil -} diff --git a/src/cmd/getM365/main.go b/src/cmd/getM365/main.go new file mode 100644 index 000000000..17aa71d78 --- /dev/null +++ b/src/cmd/getM365/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "os" + + "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cmd/getM365/exchange" + "github.com/alcionai/corso/src/cmd/getM365/onedrive" + "github.com/alcionai/corso/src/pkg/logger" +) + +var rootCmd = &cobra.Command{ + Use: "getM365", +} + +func main() { + ctx, _ := logger.SeedLevel(context.Background(), logger.Development) + + ctx = SetRootCmd(ctx, rootCmd) + defer logger.Flush(ctx) + + exchange.AddCommands(rootCmd) + onedrive.AddCommands(rootCmd) + + if err := rootCmd.Execute(); err != nil { + Err(ctx, err) + os.Exit(1) + } +} diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go new file mode 100644 index 000000000..8794fbb03 --- /dev/null +++ b/src/cmd/getM365/onedrive/get_item.go @@ -0,0 +1,207 @@ +// get_item.go is a source file designed to retrieve an m365 object from an +// existing M365 account. Data displayed is representative of the current +// serialization abstraction versioning used by Microsoft Graph and stored by Corso. + +package onedrive + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + + "github.com/alcionai/clues" + "github.com/microsoft/kiota-abstractions-go/serialization" + kjson "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/credentials" +) + +const downloadURLKey = "@microsoft.graph.downloadUrl" + +// Required inputs from user for command execution +var ( + user, tenant, m365ID string +) + +func AddCommands(parent *cobra.Command) { + exCmd := &cobra.Command{ + Use: "onedrive", + Short: "Get an M365ID item", + RunE: handleOneDriveCmd, + } + + fs := exCmd.PersistentFlags() + fs.StringVar(&m365ID, "id", "", "m365 identifier for object") + fs.StringVar(&user, "user", "", "m365 user id of M365 user") + fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") + + cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) + cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) + + parent.AddCommand(exCmd) +} + +func handleOneDriveCmd(cmd *cobra.Command, args []string) error { + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + tid := common.First(tenant, os.Getenv(account.AzureTenantID)) + + ctx := clues.Add( + cmd.Context(), + "item_id", m365ID, + "resource_owner", user, + "tenant", tid) + + // get account info + creds := account.M365Config{ + M365: credentials.GetM365(), + AzureTenantID: tid, + } + + // todo: swap to drive api client, when finished. + adpt, err := graph.CreateAdapter(tid, creds.AzureClientID, creds.AzureClientSecret) + if err != nil { + return Only(ctx, clues.Wrap(err, "creating graph adapter")) + } + + err = runDisplayM365JSON(ctx, graph.NewService(adpt), creds, user, m365ID) + if err != nil { + cmd.SilenceUsage = true + cmd.SilenceErrors = true + + return Only(ctx, clues.Wrap(err, "getting item")) + } + + return nil +} + +type itemData struct { + Size int `json:"size"` +} + +type itemPrintable struct { + Info json.RawMessage `json:"info"` + Permissions json.RawMessage `json:"permissions"` + Data itemData `json:"data"` +} + +func (i itemPrintable) MinimumPrintable() any { + return i +} + +func runDisplayM365JSON( + ctx context.Context, + srv graph.Servicer, + creds account.M365Config, + user, itemID string, +) error { + drive, err := api.GetDriveByID(ctx, srv, user) + if err != nil { + return err + } + + driveID := ptr.Val(drive.GetId()) + + it := itemPrintable{} + + item, err := api.GetDriveItem(ctx, srv, driveID, itemID) + if err != nil { + return err + } + + if item != nil { + content, err := getDriveItemContent(item) + if err != nil { + return err + } + + // We could get size from item.GetSize(), but the + // getDriveItemContent call is to ensure that we are able to + // download the file. + it.Data.Size = len(content) + } + + sInfo, err := serializeObject(item) + if err != nil { + return err + } + + err = json.Unmarshal([]byte(sInfo), &it.Info) + if err != nil { + return err + } + + perms, err := api.GetItemPermission(ctx, srv, driveID, itemID) + if err != nil { + return err + } + + sPerms, err := serializeObject(perms) + if err != nil { + return err + } + + err = json.Unmarshal([]byte(sPerms), &it.Permissions) + if err != nil { + return err + } + + PrettyJSON(ctx, it) + + return nil +} + +func serializeObject(data serialization.Parsable) (string, error) { + sw := kjson.NewJsonSerializationWriter() + + err := sw.WriteObjectValue("", data) + if err != nil { + return "", clues.Wrap(err, "writing serializing info") + } + + content, err := sw.GetSerializedContent() + if err != nil { + return "", clues.Wrap(err, "getting serializing info") + } + + return string(content), err +} + +func getDriveItemContent(item models.DriveItemable) ([]byte, error) { + url, ok := item.GetAdditionalData()[downloadURLKey].(*string) + if !ok { + return nil, clues.New("get download url") + } + + req, err := http.NewRequest(http.MethodGet, *url, nil) + if err != nil { + return nil, clues.New("create download request").With("error", err) + } + + hc := graph.HTTPClient(graph.NoTimeout()) + + resp, err := hc.Do(req) + if err != nil { + return nil, clues.New("download item").With("error", err) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return nil, clues.New("read downloaded item").With("error", err) + } + + return content, nil +} diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 4bf449044..ab2d890fc 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -29,6 +29,7 @@ import ( const ( logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS" log2xxGraphRequestsEnvKey = "LOG_2XX_GRAPH_REQUESTS" + log2xxGraphResponseEnvKey = "LOG_2XX_GRAPH_RESPONSES" retryAttemptHeader = "Retry-Attempt" retryAfterHeader = "Retry-After" defaultMaxRetries = 3 @@ -368,7 +369,7 @@ func (handler *LoggingMiddleware) Intercept( // If api logging is toggled, log a body-less dump of the request/resp. if (resp.StatusCode / 100) == 2 { if logger.DebugAPI || os.Getenv(log2xxGraphRequestsEnvKey) != "" { - log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, false)) + log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "")) } return resp, err diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 9133d6c03..f72cdf10f 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -203,10 +203,6 @@ func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) return getValues[models.Driveable](l) } -// --------------------------------------------------------------------------- -// Drive Paging -// --------------------------------------------------------------------------- - // DrivePager pages through different types of drive owners type DrivePager interface { GetPage(context.Context) (api.PageLinker, error) @@ -275,3 +271,55 @@ func GetAllDrives( return ds, nil } + +// generic drive item getter +func GetDriveItem( + ctx context.Context, + srv graph.Servicer, + driveID, itemID string, +) (models.DriveItemable, error) { + di, err := srv.Client(). + DrivesById(driveID). + ItemsById(itemID). + Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting item") + } + + return di, nil +} + +func GetItemPermission( + ctx context.Context, + service graph.Servicer, + driveID, itemID string, +) (models.PermissionCollectionResponseable, error) { + perm, err := service. + Client(). + DrivesById(driveID). + ItemsById(itemID). + Permissions(). + Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting item metadata").With("item_id", itemID) + } + + return perm, nil +} + +func GetDriveByID( + ctx context.Context, + srv graph.Servicer, + userID string, +) (models.Driveable, error) { + //revive:enable:context-as-argument + d, err := srv.Client(). + UsersById(userID). + Drive(). + Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting drive") + } + + return d, nil +} diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index 997c7c94b..6f45eb9bf 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -166,11 +167,11 @@ func NewCollection( // Allows tests to set a mock populator switch source { case SharePointSource: - c.itemGetter = getDriveItem + c.itemGetter = api.GetDriveItem c.itemReader = sharePointItemReader c.itemMetaReader = sharePointItemMetaReader default: - c.itemGetter = getDriveItem + c.itemGetter = api.GetDriveItem c.itemReader = oneDriveItemReader c.itemMetaReader = oneDriveItemMetaReader } diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 6f9a2e0ab..209cdce15 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/uploadsession" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup/details" @@ -26,20 +27,6 @@ const ( downloadURLKey = "@microsoft.graph.downloadUrl" ) -// generic drive item getter -func getDriveItem( - ctx context.Context, - srv graph.Servicer, - driveID, itemID string, -) (models.DriveItemable, error) { - di, err := srv.Client().DrivesById(driveID).ItemsById(itemID).Get(ctx, nil) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting item") - } - - return di, nil -} - // sharePointItemReader will return a io.ReadCloser for the specified item // It crafts this by querying M365 for a download URL for the item // and using a http client to initialize a reader @@ -229,14 +216,9 @@ func driveItemPermissionInfo( driveID string, itemID string, ) ([]UserPermission, error) { - perm, err := service. - Client(). - DrivesById(driveID). - ItemsById(itemID). - Permissions(). - Get(ctx, nil) + perm, err := api.GetItemPermission(ctx, service, driveID, itemID) if err != nil { - return nil, graph.Wrap(ctx, err, "fetching item permissions").With("item_id", itemID) + return nil, err } uperms := filterUserPermissions(ctx, perm.GetValue())