create onedrive custom drive (#3227)

<!-- PR description-->

Run Sanity test with custom onedrive data

#### Does this PR need a docs update or release note?
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🐛 Bugfix
- [x] 🤖 Supportability/Tests
#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
This commit is contained in:
neha_gupta 2023-04-30 15:07:53 +05:30 committed by GitHub
parent dcb9d81f3f
commit dbb950a37b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 652 additions and 47 deletions

View File

@ -31,7 +31,8 @@ jobs:
CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
CORSO_LOG_DIR: testlog CORSO_LOG_DIR: testlog
CORSO_LOG_FILE: testlog/testlogging.log CORSO_LOG_FILE: testlog/testlogging.log
CORSO_M365_TEST_USER_ID: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }} TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
SECONDARY_TEST_USER : ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
TEST_RESULT: test_results TEST_RESULT: test_results
# The default working directory doesn't seem to apply to things without # The default working directory doesn't seem to apply to things without
@ -116,7 +117,7 @@ jobs:
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
run: | run: |
go run . exchange emails \ go run . exchange emails \
--user ${{ env.CORSO_M365_TEST_USER_ID }} \ --user ${{ env.TEST_USER }} \
--tenant ${{ env.AZURE_TENANT_ID }} \ --tenant ${{ env.AZURE_TENANT_ID }} \
--destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ --destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \
--count 4 --count 4
@ -128,7 +129,7 @@ jobs:
echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE} echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE}
./corso backup create exchange \ ./corso backup create exchange \
--no-stats \ --no-stats \
--mailbox "${CORSO_M365_TEST_USER_ID}" \ --mailbox "${TEST_USER}" \
--hide-progress \ --hide-progress \
--data 'email' \ --data 'email' \
--json \ --json \
@ -136,7 +137,7 @@ jobs:
resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_exchange.txt ) resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_exchange.txt )
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
echo "backup was not successful" echo "backup was not successful"
exit 1 exit 1
fi fi
@ -209,7 +210,7 @@ jobs:
./corso backup create exchange \ ./corso backup create exchange \
--no-stats \ --no-stats \
--hide-progress \ --hide-progress \
--mailbox "${CORSO_M365_TEST_USER_ID}" \ --mailbox "${TEST_USER}" \
--json \ --json \
2>&1 | tee $TEST_RESULT/backup_exchange_incremental.txt 2>&1 | tee $TEST_RESULT/backup_exchange_incremental.txt
@ -248,6 +249,26 @@ jobs:
# Onedrive test # Onedrive test
# generate new entries for OneDrive sanity test
- name: New Data Creation for OneDrive
id: new-data-creation-onedrive
working-directory: ./src/cmd/factory
env:
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
run: |
suffix=`date +"%Y-%m-%d_%H-%M"`
go run . onedrive files \
--user ${{ env.TEST_USER }} \
--secondaryuser ${{ env.SECONDARY_TEST_USER }} \
--tenant ${{ env.AZURE_TENANT_ID }} \
--destination Corso_Restore_st_$suffix \
--count 4
echo result="$suffix" >> $GITHUB_OUTPUT
# run the tests # run the tests
- name: Backup onedrive test - name: Backup onedrive test
id: onedrive-test id: onedrive-test
@ -257,7 +278,7 @@ jobs:
./corso backup create onedrive \ ./corso backup create onedrive \
--no-stats \ --no-stats \
--hide-progress \ --hide-progress \
--user "${CORSO_M365_TEST_USER_ID}" \ --user "${TEST_USER}" \
--json \ --json \
2>&1 | tee $TEST_RESULT/backup_onedrive.txt 2>&1 | tee $TEST_RESULT/backup_onedrive.txt
@ -313,19 +334,35 @@ jobs:
./corso restore onedrive \ ./corso restore onedrive \
--no-stats \ --no-stats \
--restore-permissions \ --restore-permissions \
--folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
--hide-progress \ --hide-progress \
--backup "${{ steps.onedrive-test.outputs.result }}" \ --backup "${{ steps.onedrive-test.outputs.result }}" \
2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt 2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt
echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT
# Commenting for test cases to pass. And working on its fix - name: Restoration oneDrive check
# - name: Restoration oneDrive check env:
# env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }}
# SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive"
# SANITY_RESTORE_SERVICE: "onedrive" TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
# run: | run: |
# set -euo pipefail set -euo pipefail
# ./sanityCheck ./sanityCheck
# generate some more enteries for incremental check
- name: New Data Creation for Incremental OneDrive
working-directory: ./src/cmd/factory
env:
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
run: |
go run . onedrive files \
--user ${{ env.TEST_USER }} \
--secondaryuser ${{ env.SECONDARY_TEST_USER }} \
--tenant ${{ env.AZURE_TENANT_ID }} \
--destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
--count 4
# test onedrive incremental # test onedrive incremental
- name: Backup onedrive incremental - name: Backup onedrive incremental
@ -336,7 +373,7 @@ jobs:
./corso backup create onedrive \ ./corso backup create onedrive \
--no-stats \ --no-stats \
--hide-progress \ --hide-progress \
--user "${CORSO_M365_TEST_USER_ID}" \ --user "${TEST_USER}" \
--json \ --json \
2>&1 | tee $TEST_RESULT/backup_onedrive_incremental.txt 2>&1 | tee $TEST_RESULT/backup_onedrive_incremental.txt
@ -360,18 +397,19 @@ jobs:
--no-stats \ --no-stats \
--restore-permissions \ --restore-permissions \
--hide-progress \ --hide-progress \
--folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
--backup "${{ steps.onedrive-incremental-test.outputs.result }}" \ --backup "${{ steps.onedrive-incremental-test.outputs.result }}" \
2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt 2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt
echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT
# Commenting for test cases to pass. And working on its fix - name: Restoration oneDrive check
# - name: Restoration oneDrive check env:
# env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }}
# SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive"
# SANITY_RESTORE_SERVICE: "onedrive" TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
# run: | run: |
# set -euo pipefail set -euo pipefail
# ./sanityCheck ./sanityCheck
# Upload the original go test output as an artifact for later review. # Upload the original go test output as an artifact for later review.
- name: Upload test log - name: Upload test log

View File

@ -44,6 +44,7 @@ func main() {
fs.StringVar(&impl.Tenant, "tenant", "", "m365 tenant containing the user") fs.StringVar(&impl.Tenant, "tenant", "", "m365 tenant containing the user")
fs.StringVar(&impl.User, "user", "", "m365 user owning the new data") fs.StringVar(&impl.User, "user", "", "m365 user owning the new data")
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("user")) cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("user"))
fs.StringVar(&impl.SecondaryUser, "secondaryuser", "", "m365 secondary user owning the new data")
fs.IntVar(&impl.Count, "count", 0, "count of items to produce") fs.IntVar(&impl.Count, "count", 0, "count of items to produce")
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("count")) cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("count"))
fs.StringVar(&impl.Destination, "destination", "", "destination of the new data (will create as needed)") fs.StringVar(&impl.Destination, "destination", "", "destination of the new data (will create as needed)")

View File

@ -1,8 +1,13 @@
package impl package impl
import ( import (
"bytes"
"context" "context"
"encoding/json"
"fmt"
"io"
"os" "os"
"strings"
"time" "time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -10,9 +15,14 @@ import (
"github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector"
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
@ -24,10 +34,11 @@ import (
) )
var ( var (
Count int Count int
Destination string Destination string
Tenant string Tenant string
User string User string
SecondaryUser string
) )
// TODO: ErrGenerating = clues.New("not all items were successfully generated") // TODO: ErrGenerating = clues.New("not all items were successfully generated")
@ -76,7 +87,6 @@ func generateAndRestoreItems(
items: items, items: items,
}} }}
// TODO: fit the destination to the containers
dest := control.DefaultRestoreDestination(common.SimpleTimeTesting) dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
dest.ContainerName = destFldr dest.ContainerName = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName) print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
@ -99,7 +109,15 @@ func generateAndRestoreItems(
// Common Helpers // Common Helpers
// ------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------
func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, account.Account, error) { func getGCAndVerifyUser(
ctx context.Context,
userID string,
) (
*connector.GraphConnector,
account.Account,
idname.Provider,
error,
) {
tid := common.First(Tenant, os.Getenv(account.AzureTenantID)) tid := common.First(Tenant, os.Getenv(account.AzureTenantID))
if len(Tenant) == 0 { if len(Tenant) == 0 {
@ -114,7 +132,7 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
acct, err := account.NewAccount(account.ProviderM365, m365Cfg) acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil { if err != nil {
return nil, account.Account{}, clues.Wrap(err, "finding m365 account details") return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
} }
gc, err := connector.NewGraphConnector( gc, err := connector.NewGraphConnector(
@ -122,14 +140,15 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
acct, acct,
connector.Users) connector.Users)
if err != nil { if err != nil {
return nil, account.Account{}, clues.Wrap(err, "connecting to graph api") return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
} }
if _, _, err := gc.PopulateOwnerIDAndNamesFrom(ctx, userID, nil); err != nil { id, _, err := gc.PopulateOwnerIDAndNamesFrom(ctx, userID, nil)
return nil, account.Account{}, clues.Wrap(err, "verifying user") if err != nil {
return nil, account.Account{}, nil, clues.Wrap(err, "verifying user")
} }
return gc, acct, nil return gc, acct, gc.IDNameLookup.ProviderForID(id), nil
} }
type item struct { type item struct {
@ -179,3 +198,504 @@ func buildCollections(
return collections, nil return collections, nil
} }
type permData struct {
user string // user is only for older versions
entityID string
roles []string
sharingMode onedrive.SharingMode
}
type itemData struct {
name string
data []byte
perms permData
}
type itemInfo struct {
// lookupKey is a string that can be used to find this data from a set of
// other data in the same collection. This key should be something that will
// be the same before and after restoring the item in M365 and may not be
// the M365 ID. When restoring items out of place, the item is assigned a
// new ID making it unsuitable for a lookup key.
lookupKey string
name string
data []byte
}
type onedriveCollection struct {
service path.ServiceType
pathElements []string
items []itemInfo
aux []itemInfo
backupVersion int
}
type onedriveColInfo struct {
pathElements []string
perms permData
files []itemData
folders []itemData
}
var (
folderAName = "folder-a"
folderBName = "b"
folderCName = "folder-c"
fileAData = []byte(strings.Repeat("a", 33))
fileBData = []byte(strings.Repeat("b", 65))
fileEData = []byte(strings.Repeat("e", 257))
// Cannot restore owner or empty permissions and so not testing them
writePerm = []string{"write"}
readPerm = []string{"read"}
)
func generateAndRestoreOnedriveItems(
gc *connector.GraphConnector,
resourceOwner, secondaryUserID, secondaryUserName string,
acct account.Account,
service path.ServiceType,
cat path.CategoryType,
sel selectors.Selector,
tenantID, destFldr string,
count int,
errs *fault.Bus,
) (
*details.Details,
error,
) {
ctx, flush := tester.NewContext()
defer flush()
dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
dest.ContainerName = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
d, _ := gc.Service.Client().UsersById(resourceOwner).Drive().Get(ctx, nil)
driveID := ptr.Val(d.GetId())
var (
cols []onedriveColInfo
rootPath = []string{"drives", driveID, "root:"}
folderAPath = []string{"drives", driveID, "root:", folderAName}
folderBPath = []string{"drives", driveID, "root:", folderBName}
folderCPath = []string{"drives", driveID, "root:", folderCName}
now = time.Now()
year, mnth, date = now.Date()
hour, min, sec = now.Clock()
currentTime = fmt.Sprintf("%d-%v-%d-%d-%d-%d", year, mnth, date, hour, min, sec)
)
for i := 0; i < count; i++ {
col := []onedriveColInfo{
// basic folder and file creation
{
pathElements: rootPath,
files: []itemData{
{
name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
data: fileAData,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: writePerm,
},
},
{
name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime),
data: fileBData,
},
},
folders: []itemData{
{
name: folderBName,
},
{
name: folderAName,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: readPerm,
},
},
{
name: folderCName,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: readPerm,
},
},
},
},
{
// a folder that has permissions with an item in the folder with
// the different permissions.
pathElements: folderAPath,
files: []itemData{
{
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileEData,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: writePerm,
},
},
},
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: readPerm,
},
},
{
// a folder that has permissions with an item in the folder with
// no permissions.
pathElements: folderCPath,
files: []itemData{
{
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileAData,
},
},
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: readPerm,
},
},
{
pathElements: folderBPath,
files: []itemData{
{
// restoring a file in a non-root folder that doesn't inherit
// permissions.
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
data: fileBData,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: writePerm,
},
},
},
folders: []itemData{
{
name: folderAName,
perms: permData{
user: secondaryUserName,
entityID: secondaryUserID,
roles: readPerm,
},
},
},
},
}
cols = append(cols, col...)
}
input := dataForInfo(service, cols, version.Backup)
collections := getCollections(
service,
tenantID,
[]string{resourceOwner},
input,
version.Backup)
opts := control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
}
return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, collections, errs)
}
func getCollections(
service path.ServiceType,
tenant string,
resourceOwners []string,
testCollections []colInfo,
backupVersion int,
) []data.RestoreCollection {
var collections []data.RestoreCollection
for _, owner := range resourceOwners {
ownerCollections := collectionsForInfo(
service,
tenant,
owner,
testCollections,
backupVersion,
)
collections = append(collections, ownerCollections...)
}
return collections
}
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) Fetch(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
service path.ServiceType,
tenant, user string,
allInfo []colInfo,
backupVersion int,
) []data.RestoreCollection {
collections := make([]data.RestoreCollection, 0, len(allInfo))
for _, info := range allInfo {
pth := mustToDataLayerPath(
service,
tenant,
user,
info.category,
info.pathElements,
false)
mc := exchMock.NewCollection(pth, pth, len(info.items))
for i := 0; i < len(info.items); i++ {
mc.Names[i] = info.items[i].name
mc.Data[i] = info.items[i].data
// We do not count metadata files against item count
if backupVersion > 0 && metadata.HasMetaSuffix(info.items[i].name) &&
(service == path.OneDriveService || service == path.SharePointService) {
continue
}
}
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.auxItems {
c.auxItems[aux.name] = &exchMock.Data{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
}
return collections
}
func mustToDataLayerPath(
service path.ServiceType,
tenant, resourceOwner string,
category path.CategoryType,
elements []string,
isItem bool,
) path.Path {
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
if err != nil {
fmt.Println("building path", clues.ToCore(err))
}
return res
}
type colInfo struct {
// Elements (in order) for the path representing this collection. Should
// only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be
// "Inbox".
pathElements []string
category path.CategoryType
items []itemInfo
// auxItems are items that can be retrieved with Fetch but won't be returned
// by Items().
auxItems []itemInfo
}
func newOneDriveCollection(
service path.ServiceType,
pathElements []string,
backupVersion int,
) *onedriveCollection {
return &onedriveCollection{
service: service,
pathElements: pathElements,
backupVersion: backupVersion,
}
}
func dataForInfo(
service path.ServiceType,
cols []onedriveColInfo,
backupVersion int,
) []colInfo {
var res []colInfo
for _, c := range cols {
onedriveCol := newOneDriveCollection(service, c.pathElements, backupVersion)
for _, f := range c.files {
onedriveCol.withFile(f.name, f.data, f.perms)
}
onedriveCol.withPermissions(c.perms)
res = append(res, onedriveCol.collection())
}
return res
}
func (c onedriveCollection) collection() colInfo {
cat := path.FilesCategory
if c.service == path.SharePointService {
cat = path.LibrariesCategory
}
return colInfo{
pathElements: c.pathElements,
category: cat,
items: c.items,
auxItems: c.aux,
}
}
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection {
c.items = append(c.items, onedriveItemWithData(
name+metadata.DataFileSuffix,
name+metadata.DataFileSuffix,
fileData))
md := onedriveMetadata(
name,
name+metadata.MetaFileSuffix,
name,
perm,
true)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
return c
}
// withPermissions adds permissions to the folder represented by this
// onedriveCollection.
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection {
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
return c
}
name := c.pathElements[len(c.pathElements)-1]
metaName := name
if c.backupVersion >= version.OneDrive5DirMetaNoName {
// We switched to just .dirmeta for metadata file names.
metaName = ""
}
if name == "root:" {
return c
}
md := onedriveMetadata(
name,
metaName+metadata.DirMetaFileSuffix,
metaName+metadata.DirMetaFileSuffix,
perm,
true)
c.items = append(c.items, md)
c.aux = append(c.aux, md)
return c
}
type oneDriveData struct {
FileName string `json:"fileName,omitempty"`
Data []byte `json:"data,omitempty"`
}
func onedriveItemWithData(
name, lookupKey string,
fileData []byte,
) itemInfo {
content := oneDriveData{
FileName: lookupKey,
Data: fileData,
}
serialized, _ := json.Marshal(content)
return itemInfo{
name: name,
data: serialized,
lookupKey: lookupKey,
}
}
func onedriveMetadata(
fileName, itemID, lookupKey string,
perm permData,
permUseID bool,
) itemInfo {
meta := getMetadata(fileName, perm, permUseID)
metaJSON, err := json.Marshal(meta)
if err != nil {
fmt.Println("marshalling metadata", clues.ToCore(err))
}
return itemInfo{
name: itemID,
data: metaJSON,
lookupKey: lookupKey,
}
}
func getMetadata(fileName string, perm permData, permUseID bool) onedrive.Metadata {
if len(perm.user) == 0 || len(perm.roles) == 0 ||
perm.sharingMode != onedrive.SharingModeCustom {
return onedrive.Metadata{
FileName: fileName,
SharingMode: perm.sharingMode,
}
}
// In case of permissions, the id will usually be same for same
// user/role combo unless deleted and readded, but we have to do
// this as we only have two users of which one is already taken.
id := uuid.NewString()
uperm := onedrive.UserPermission{ID: id, Roles: perm.roles}
if permUseID {
uperm.EntityID = perm.entityID
} else {
uperm.Email = perm.user
}
meta := onedrive.Metadata{
FileName: fileName,
Permissions: []onedrive.UserPermission{uperm},
}
return meta
}

View File

@ -51,7 +51,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
return nil return nil
} }
gc, acct, err := getGCAndVerifyUser(ctx, User) gc, acct, _, err := getGCAndVerifyUser(ctx, User)
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -98,7 +98,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
return nil return nil
} }
gc, acct, err := getGCAndVerifyUser(ctx, User) gc, acct, _, err := getGCAndVerifyUser(ctx, User)
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -144,7 +144,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
return nil return nil
} }
gc, acct, err := getGCAndVerifyUser(ctx, User) gc, acct, _, err := getGCAndVerifyUser(ctx, User)
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -1,10 +1,16 @@
package impl package impl
import ( import (
"strings"
"github.com/spf13/cobra" "github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
) )
var filesCmd = &cobra.Command{ var filesCmd = &cobra.Command{
@ -18,11 +24,44 @@ func AddOneDriveCommands(cmd *cobra.Command) {
} }
func handleOneDriveFileFactory(cmd *cobra.Command, args []string) error { func handleOneDriveFileFactory(cmd *cobra.Command, args []string) error {
Err(cmd.Context(), ErrNotYetImplemented) var (
ctx = cmd.Context()
service = path.OneDriveService
category = path.FilesCategory
errs = fault.New(false)
)
if utils.HasNoFlagsAndShownHelp(cmd) { if utils.HasNoFlagsAndShownHelp(cmd) {
return nil return nil
} }
gc, acct, inp, err := getGCAndVerifyUser(ctx, User)
if err != nil {
return Only(ctx, err)
}
deets, err := generateAndRestoreOnedriveItems(
gc,
User,
inp.ID(),
strings.ToLower(SecondaryUser),
acct,
service,
category,
selectors.NewOneDriveBackup([]string{User}).Selector,
Tenant,
Destination,
Count,
errs)
if err != nil {
return Only(ctx, err)
}
for _, e := range errs.Recovered() {
logger.CtxErr(ctx, err).Error(e.Error())
}
deets.PrintEntries(ctx)
return nil return nil
} }

View File

@ -83,7 +83,7 @@ func main() {
case "exchange": case "exchange":
checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
case "onedrive": case "onedrive":
checkOnedriveRestoration(ctx, client, testUser, folder, startTime) checkOnedriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
default: default:
fatal(ctx, "no service specified", nil) fatal(ctx, "no service specified", nil)
} }
@ -296,7 +296,7 @@ func checkOnedriveRestoration(
ctx context.Context, ctx context.Context,
client *msgraphsdk.GraphServiceClient, client *msgraphsdk.GraphServiceClient,
testUser, testUser,
folderName string, folderName, dataFolder string,
startTime time.Time, startTime time.Time,
) { ) {
var ( var (
@ -337,7 +337,6 @@ func checkOnedriveRestoration(
var ( var (
itemID = ptr.Val(driveItem.GetId()) itemID = ptr.Val(driveItem.GetId())
itemName = ptr.Val(driveItem.GetName()) itemName = ptr.Val(driveItem.GetName())
ictx = clues.Add(ctx, "item_id", itemID, "item_name", itemName)
) )
if itemName == folderName { if itemName == folderName {
@ -345,8 +344,10 @@ func checkOnedriveRestoration(
continue continue
} }
folderTime, hasTime := mustGetTimeFromName(ictx, itemName) if itemName != dataFolder {
if !isWithinTimeBound(ctx, startTime, folderTime, hasTime) { logger.Ctx(ctx).Infof("test data for %v folder: ", dataFolder)
fmt.Printf("test data for %v folder: ", dataFolder)
continue continue
} }
@ -375,8 +376,7 @@ func checkOnedriveRestoration(
getRestoredDrive(ctx, client, *drive.GetId(), restoreFolderID, restoreFile, restoreFolderPermission, startTime) getRestoredDrive(ctx, client, *drive.GetId(), restoreFolderID, restoreFile, restoreFolderPermission, startTime)
for folderName, permissions := range folderPermission { for folderName, permissions := range folderPermission {
logger.Ctx(ctx).Info("checking for folder: ", folderName) logAndPrint(ctx, "checking for folder: %s", folderName)
fmt.Printf("checking for folder: %s\n", folderName)
restoreFolderPerm := restoreFolderPermission[folderName] restoreFolderPerm := restoreFolderPermission[folderName]
@ -415,6 +415,9 @@ func checkOnedriveRestoration(
} }
for fileName, expected := range fileSizes { for fileName, expected := range fileSizes {
logger.Ctx(ctx).Info("checking for file: ", fileName)
fmt.Printf("checking for file: %s\n", fileName)
got := restoreFile[fileName] got := restoreFile[fileName]
assert( assert(
@ -465,8 +468,7 @@ func getOneDriveChildFolder(
// currently we don't restore blank folders. // currently we don't restore blank folders.
// skip permission check for empty folders // skip permission check for empty folders
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
logger.Ctx(ctx).Info("skipped empty folder: ", fullName) logAndPrint(ctx, "skipped empty folder: %s", fullName)
fmt.Println("skipped empty folder: ", fullName)
continue continue
} }
@ -633,3 +635,8 @@ func assert(
os.Exit(1) os.Exit(1)
} }
func logAndPrint(ctx context.Context, tmpl string, vs ...any) {
logger.Ctx(ctx).Infof(tmpl, vs...)
fmt.Printf(tmpl+"\n", vs...)
}