create onedrive custom drive (#3227)
<!-- PR description--> Run Sanity test with custom onedrive data #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🐛 Bugfix - [x] 🤖 Supportability/Tests #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #<issue> #### Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual
This commit is contained in:
parent
dcb9d81f3f
commit
dbb950a37b
84
.github/workflows/sanity-test.yaml
vendored
84
.github/workflows/sanity-test.yaml
vendored
@ -31,7 +31,8 @@ jobs:
|
||||
CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
|
||||
CORSO_LOG_DIR: testlog
|
||||
CORSO_LOG_FILE: testlog/testlogging.log
|
||||
CORSO_M365_TEST_USER_ID: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
|
||||
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
|
||||
SECONDARY_TEST_USER : ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
|
||||
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
|
||||
TEST_RESULT: test_results
|
||||
# The default working directory doesn't seem to apply to things without
|
||||
@ -116,7 +117,7 @@ jobs:
|
||||
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
run: |
|
||||
go run . exchange emails \
|
||||
--user ${{ env.CORSO_M365_TEST_USER_ID }} \
|
||||
--user ${{ env.TEST_USER }} \
|
||||
--tenant ${{ env.AZURE_TENANT_ID }} \
|
||||
--destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \
|
||||
--count 4
|
||||
@ -128,7 +129,7 @@ jobs:
|
||||
echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE}
|
||||
./corso backup create exchange \
|
||||
--no-stats \
|
||||
--mailbox "${CORSO_M365_TEST_USER_ID}" \
|
||||
--mailbox "${TEST_USER}" \
|
||||
--hide-progress \
|
||||
--data 'email' \
|
||||
--json \
|
||||
@ -136,7 +137,7 @@ jobs:
|
||||
|
||||
resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_exchange.txt )
|
||||
|
||||
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
|
||||
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
|
||||
echo "backup was not successful"
|
||||
exit 1
|
||||
fi
|
||||
@ -209,7 +210,7 @@ jobs:
|
||||
./corso backup create exchange \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
--mailbox "${CORSO_M365_TEST_USER_ID}" \
|
||||
--mailbox "${TEST_USER}" \
|
||||
--json \
|
||||
2>&1 | tee $TEST_RESULT/backup_exchange_incremental.txt
|
||||
|
||||
@ -248,6 +249,26 @@ jobs:
|
||||
|
||||
# Onedrive test
|
||||
|
||||
# generate new entries for OneDrive sanity test
|
||||
- name: New Data Creation for OneDrive
|
||||
id: new-data-creation-onedrive
|
||||
working-directory: ./src/cmd/factory
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
|
||||
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
run: |
|
||||
suffix=`date +"%Y-%m-%d_%H-%M"`
|
||||
|
||||
go run . onedrive files \
|
||||
--user ${{ env.TEST_USER }} \
|
||||
--secondaryuser ${{ env.SECONDARY_TEST_USER }} \
|
||||
--tenant ${{ env.AZURE_TENANT_ID }} \
|
||||
--destination Corso_Restore_st_$suffix \
|
||||
--count 4
|
||||
|
||||
echo result="$suffix" >> $GITHUB_OUTPUT
|
||||
|
||||
# run the tests
|
||||
- name: Backup onedrive test
|
||||
id: onedrive-test
|
||||
@ -257,7 +278,7 @@ jobs:
|
||||
./corso backup create onedrive \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
--user "${CORSO_M365_TEST_USER_ID}" \
|
||||
--user "${TEST_USER}" \
|
||||
--json \
|
||||
2>&1 | tee $TEST_RESULT/backup_onedrive.txt
|
||||
|
||||
@ -313,19 +334,35 @@ jobs:
|
||||
./corso restore onedrive \
|
||||
--no-stats \
|
||||
--restore-permissions \
|
||||
--folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
|
||||
--hide-progress \
|
||||
--backup "${{ steps.onedrive-test.outputs.result }}" \
|
||||
2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt
|
||||
echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT
|
||||
|
||||
# Commenting for test cases to pass. And working on its fix
|
||||
# - name: Restoration oneDrive check
|
||||
# env:
|
||||
# SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }}
|
||||
# SANITY_RESTORE_SERVICE: "onedrive"
|
||||
# run: |
|
||||
# set -euo pipefail
|
||||
# ./sanityCheck
|
||||
- name: Restoration oneDrive check
|
||||
env:
|
||||
SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }}
|
||||
SANITY_RESTORE_SERVICE: "onedrive"
|
||||
TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
./sanityCheck
|
||||
|
||||
# generate some more enteries for incremental check
|
||||
- name: New Data Creation for Incremental OneDrive
|
||||
working-directory: ./src/cmd/factory
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
|
||||
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
run: |
|
||||
go run . onedrive files \
|
||||
--user ${{ env.TEST_USER }} \
|
||||
--secondaryuser ${{ env.SECONDARY_TEST_USER }} \
|
||||
--tenant ${{ env.AZURE_TENANT_ID }} \
|
||||
--destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
|
||||
--count 4
|
||||
|
||||
# test onedrive incremental
|
||||
- name: Backup onedrive incremental
|
||||
@ -336,7 +373,7 @@ jobs:
|
||||
./corso backup create onedrive \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
--user "${CORSO_M365_TEST_USER_ID}" \
|
||||
--user "${TEST_USER}" \
|
||||
--json \
|
||||
2>&1 | tee $TEST_RESULT/backup_onedrive_incremental.txt
|
||||
|
||||
@ -360,18 +397,19 @@ jobs:
|
||||
--no-stats \
|
||||
--restore-permissions \
|
||||
--hide-progress \
|
||||
--folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
|
||||
--backup "${{ steps.onedrive-incremental-test.outputs.result }}" \
|
||||
2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt
|
||||
echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT
|
||||
|
||||
# Commenting for test cases to pass. And working on its fix
|
||||
# - name: Restoration oneDrive check
|
||||
# env:
|
||||
# SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }}
|
||||
# SANITY_RESTORE_SERVICE: "onedrive"
|
||||
# run: |
|
||||
# set -euo pipefail
|
||||
# ./sanityCheck
|
||||
- name: Restoration oneDrive check
|
||||
env:
|
||||
SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }}
|
||||
SANITY_RESTORE_SERVICE: "onedrive"
|
||||
TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
./sanityCheck
|
||||
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
|
||||
@ -44,6 +44,7 @@ func main() {
|
||||
fs.StringVar(&impl.Tenant, "tenant", "", "m365 tenant containing the user")
|
||||
fs.StringVar(&impl.User, "user", "", "m365 user owning the new data")
|
||||
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("user"))
|
||||
fs.StringVar(&impl.SecondaryUser, "secondaryuser", "", "m365 secondary user owning the new data")
|
||||
fs.IntVar(&impl.Count, "count", 0, "count of items to produce")
|
||||
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("count"))
|
||||
fs.StringVar(&impl.Destination, "destination", "", "destination of the new data (will create as needed)")
|
||||
|
||||
@ -1,8 +1,13 @@
|
||||
package impl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
@ -10,9 +15,14 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/internal/common"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/connector"
|
||||
exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
@ -24,10 +34,11 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
Count int
|
||||
Destination string
|
||||
Tenant string
|
||||
User string
|
||||
Count int
|
||||
Destination string
|
||||
Tenant string
|
||||
User string
|
||||
SecondaryUser string
|
||||
)
|
||||
|
||||
// TODO: ErrGenerating = clues.New("not all items were successfully generated")
|
||||
@ -76,7 +87,6 @@ func generateAndRestoreItems(
|
||||
items: items,
|
||||
}}
|
||||
|
||||
// TODO: fit the destination to the containers
|
||||
dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
|
||||
dest.ContainerName = destFldr
|
||||
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
|
||||
@ -99,7 +109,15 @@ func generateAndRestoreItems(
|
||||
// Common Helpers
|
||||
// ------------------------------------------------------------------------------------------
|
||||
|
||||
func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, account.Account, error) {
|
||||
func getGCAndVerifyUser(
|
||||
ctx context.Context,
|
||||
userID string,
|
||||
) (
|
||||
*connector.GraphConnector,
|
||||
account.Account,
|
||||
idname.Provider,
|
||||
error,
|
||||
) {
|
||||
tid := common.First(Tenant, os.Getenv(account.AzureTenantID))
|
||||
|
||||
if len(Tenant) == 0 {
|
||||
@ -114,7 +132,7 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
|
||||
|
||||
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
|
||||
if err != nil {
|
||||
return nil, account.Account{}, clues.Wrap(err, "finding m365 account details")
|
||||
return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
|
||||
}
|
||||
|
||||
gc, err := connector.NewGraphConnector(
|
||||
@ -122,14 +140,15 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
|
||||
acct,
|
||||
connector.Users)
|
||||
if err != nil {
|
||||
return nil, account.Account{}, clues.Wrap(err, "connecting to graph api")
|
||||
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
|
||||
}
|
||||
|
||||
if _, _, err := gc.PopulateOwnerIDAndNamesFrom(ctx, userID, nil); err != nil {
|
||||
return nil, account.Account{}, clues.Wrap(err, "verifying user")
|
||||
id, _, err := gc.PopulateOwnerIDAndNamesFrom(ctx, userID, nil)
|
||||
if err != nil {
|
||||
return nil, account.Account{}, nil, clues.Wrap(err, "verifying user")
|
||||
}
|
||||
|
||||
return gc, acct, nil
|
||||
return gc, acct, gc.IDNameLookup.ProviderForID(id), nil
|
||||
}
|
||||
|
||||
type item struct {
|
||||
@ -179,3 +198,504 @@ func buildCollections(
|
||||
|
||||
return collections, nil
|
||||
}
|
||||
|
||||
type permData struct {
|
||||
user string // user is only for older versions
|
||||
entityID string
|
||||
roles []string
|
||||
sharingMode onedrive.SharingMode
|
||||
}
|
||||
|
||||
type itemData struct {
|
||||
name string
|
||||
data []byte
|
||||
perms permData
|
||||
}
|
||||
|
||||
type itemInfo struct {
|
||||
// lookupKey is a string that can be used to find this data from a set of
|
||||
// other data in the same collection. This key should be something that will
|
||||
// be the same before and after restoring the item in M365 and may not be
|
||||
// the M365 ID. When restoring items out of place, the item is assigned a
|
||||
// new ID making it unsuitable for a lookup key.
|
||||
lookupKey string
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type onedriveCollection struct {
|
||||
service path.ServiceType
|
||||
pathElements []string
|
||||
items []itemInfo
|
||||
aux []itemInfo
|
||||
backupVersion int
|
||||
}
|
||||
|
||||
type onedriveColInfo struct {
|
||||
pathElements []string
|
||||
perms permData
|
||||
files []itemData
|
||||
folders []itemData
|
||||
}
|
||||
|
||||
var (
|
||||
folderAName = "folder-a"
|
||||
folderBName = "b"
|
||||
folderCName = "folder-c"
|
||||
|
||||
fileAData = []byte(strings.Repeat("a", 33))
|
||||
fileBData = []byte(strings.Repeat("b", 65))
|
||||
fileEData = []byte(strings.Repeat("e", 257))
|
||||
|
||||
// Cannot restore owner or empty permissions and so not testing them
|
||||
writePerm = []string{"write"}
|
||||
readPerm = []string{"read"}
|
||||
)
|
||||
|
||||
func generateAndRestoreOnedriveItems(
|
||||
gc *connector.GraphConnector,
|
||||
resourceOwner, secondaryUserID, secondaryUserName string,
|
||||
acct account.Account,
|
||||
service path.ServiceType,
|
||||
cat path.CategoryType,
|
||||
sel selectors.Selector,
|
||||
tenantID, destFldr string,
|
||||
count int,
|
||||
errs *fault.Bus,
|
||||
) (
|
||||
*details.Details,
|
||||
error,
|
||||
) {
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
|
||||
dest.ContainerName = destFldr
|
||||
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
|
||||
|
||||
d, _ := gc.Service.Client().UsersById(resourceOwner).Drive().Get(ctx, nil)
|
||||
driveID := ptr.Val(d.GetId())
|
||||
|
||||
var (
|
||||
cols []onedriveColInfo
|
||||
|
||||
rootPath = []string{"drives", driveID, "root:"}
|
||||
folderAPath = []string{"drives", driveID, "root:", folderAName}
|
||||
folderBPath = []string{"drives", driveID, "root:", folderBName}
|
||||
folderCPath = []string{"drives", driveID, "root:", folderCName}
|
||||
|
||||
now = time.Now()
|
||||
year, mnth, date = now.Date()
|
||||
hour, min, sec = now.Clock()
|
||||
currentTime = fmt.Sprintf("%d-%v-%d-%d-%d-%d", year, mnth, date, hour, min, sec)
|
||||
)
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
col := []onedriveColInfo{
|
||||
// basic folder and file creation
|
||||
{
|
||||
pathElements: rootPath,
|
||||
files: []itemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-1st-count-%d-at-%s", i, currentTime),
|
||||
data: fileAData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: fmt.Sprintf("file-2nd-count-%d-at-%s", i, currentTime),
|
||||
data: fileBData,
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
{
|
||||
name: folderBName,
|
||||
},
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: folderCName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// a folder that has permissions with an item in the folder with
|
||||
// the different permissions.
|
||||
pathElements: folderAPath,
|
||||
files: []itemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileEData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
// a folder that has permissions with an item in the folder with
|
||||
// no permissions.
|
||||
pathElements: folderCPath,
|
||||
files: []itemData{
|
||||
{
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileAData,
|
||||
},
|
||||
},
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
},
|
||||
},
|
||||
{
|
||||
pathElements: folderBPath,
|
||||
files: []itemData{
|
||||
{
|
||||
// restoring a file in a non-root folder that doesn't inherit
|
||||
// permissions.
|
||||
name: fmt.Sprintf("file-count-%d-at-%s", i, currentTime),
|
||||
data: fileBData,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: writePerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
folders: []itemData{
|
||||
{
|
||||
name: folderAName,
|
||||
perms: permData{
|
||||
user: secondaryUserName,
|
||||
entityID: secondaryUserID,
|
||||
roles: readPerm,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cols = append(cols, col...)
|
||||
}
|
||||
|
||||
input := dataForInfo(service, cols, version.Backup)
|
||||
|
||||
collections := getCollections(
|
||||
service,
|
||||
tenantID,
|
||||
[]string{resourceOwner},
|
||||
input,
|
||||
version.Backup)
|
||||
|
||||
opts := control.Options{
|
||||
RestorePermissions: true,
|
||||
ToggleFeatures: control.Toggles{},
|
||||
}
|
||||
|
||||
return gc.ConsumeRestoreCollections(ctx, version.Backup, acct, sel, dest, opts, collections, errs)
|
||||
}
|
||||
|
||||
func getCollections(
|
||||
service path.ServiceType,
|
||||
tenant string,
|
||||
resourceOwners []string,
|
||||
testCollections []colInfo,
|
||||
backupVersion int,
|
||||
) []data.RestoreCollection {
|
||||
var collections []data.RestoreCollection
|
||||
|
||||
for _, owner := range resourceOwners {
|
||||
ownerCollections := collectionsForInfo(
|
||||
service,
|
||||
tenant,
|
||||
owner,
|
||||
testCollections,
|
||||
backupVersion,
|
||||
)
|
||||
|
||||
collections = append(collections, ownerCollections...)
|
||||
}
|
||||
|
||||
return collections
|
||||
}
|
||||
|
||||
type mockRestoreCollection struct {
|
||||
data.Collection
|
||||
auxItems map[string]data.Stream
|
||||
}
|
||||
|
||||
func (rc mockRestoreCollection) Fetch(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
) (data.Stream, error) {
|
||||
res := rc.auxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func collectionsForInfo(
|
||||
service path.ServiceType,
|
||||
tenant, user string,
|
||||
allInfo []colInfo,
|
||||
backupVersion int,
|
||||
) []data.RestoreCollection {
|
||||
collections := make([]data.RestoreCollection, 0, len(allInfo))
|
||||
|
||||
for _, info := range allInfo {
|
||||
pth := mustToDataLayerPath(
|
||||
service,
|
||||
tenant,
|
||||
user,
|
||||
info.category,
|
||||
info.pathElements,
|
||||
false)
|
||||
|
||||
mc := exchMock.NewCollection(pth, pth, len(info.items))
|
||||
|
||||
for i := 0; i < len(info.items); i++ {
|
||||
mc.Names[i] = info.items[i].name
|
||||
mc.Data[i] = info.items[i].data
|
||||
|
||||
// We do not count metadata files against item count
|
||||
if backupVersion > 0 && metadata.HasMetaSuffix(info.items[i].name) &&
|
||||
(service == path.OneDriveService || service == path.SharePointService) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
|
||||
|
||||
for _, aux := range info.auxItems {
|
||||
c.auxItems[aux.name] = &exchMock.Data{
|
||||
ID: aux.name,
|
||||
Reader: io.NopCloser(bytes.NewReader(aux.data)),
|
||||
}
|
||||
}
|
||||
|
||||
collections = append(collections, c)
|
||||
}
|
||||
|
||||
return collections
|
||||
}
|
||||
|
||||
func mustToDataLayerPath(
|
||||
service path.ServiceType,
|
||||
tenant, resourceOwner string,
|
||||
category path.CategoryType,
|
||||
elements []string,
|
||||
isItem bool,
|
||||
) path.Path {
|
||||
res, err := path.Build(tenant, resourceOwner, service, category, isItem, elements...)
|
||||
if err != nil {
|
||||
fmt.Println("building path", clues.ToCore(err))
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
type colInfo struct {
|
||||
// Elements (in order) for the path representing this collection. Should
|
||||
// only contain elements after the prefix that corso uses for the path. For
|
||||
// example, a collection for the Inbox folder in exchange mail would just be
|
||||
// "Inbox".
|
||||
pathElements []string
|
||||
category path.CategoryType
|
||||
items []itemInfo
|
||||
// auxItems are items that can be retrieved with Fetch but won't be returned
|
||||
// by Items().
|
||||
auxItems []itemInfo
|
||||
}
|
||||
|
||||
func newOneDriveCollection(
|
||||
service path.ServiceType,
|
||||
pathElements []string,
|
||||
backupVersion int,
|
||||
) *onedriveCollection {
|
||||
return &onedriveCollection{
|
||||
service: service,
|
||||
pathElements: pathElements,
|
||||
backupVersion: backupVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func dataForInfo(
|
||||
service path.ServiceType,
|
||||
cols []onedriveColInfo,
|
||||
backupVersion int,
|
||||
) []colInfo {
|
||||
var res []colInfo
|
||||
|
||||
for _, c := range cols {
|
||||
onedriveCol := newOneDriveCollection(service, c.pathElements, backupVersion)
|
||||
|
||||
for _, f := range c.files {
|
||||
onedriveCol.withFile(f.name, f.data, f.perms)
|
||||
}
|
||||
|
||||
onedriveCol.withPermissions(c.perms)
|
||||
|
||||
res = append(res, onedriveCol.collection())
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (c onedriveCollection) collection() colInfo {
|
||||
cat := path.FilesCategory
|
||||
if c.service == path.SharePointService {
|
||||
cat = path.LibrariesCategory
|
||||
}
|
||||
|
||||
return colInfo{
|
||||
pathElements: c.pathElements,
|
||||
category: cat,
|
||||
items: c.items,
|
||||
auxItems: c.aux,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *onedriveCollection) withFile(name string, fileData []byte, perm permData) *onedriveCollection {
|
||||
c.items = append(c.items, onedriveItemWithData(
|
||||
name+metadata.DataFileSuffix,
|
||||
name+metadata.DataFileSuffix,
|
||||
fileData))
|
||||
|
||||
md := onedriveMetadata(
|
||||
name,
|
||||
name+metadata.MetaFileSuffix,
|
||||
name,
|
||||
perm,
|
||||
true)
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// withPermissions adds permissions to the folder represented by this
|
||||
// onedriveCollection.
|
||||
func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection {
|
||||
if c.backupVersion < version.OneDrive4DirIncludesPermissions {
|
||||
return c
|
||||
}
|
||||
|
||||
name := c.pathElements[len(c.pathElements)-1]
|
||||
metaName := name
|
||||
|
||||
if c.backupVersion >= version.OneDrive5DirMetaNoName {
|
||||
// We switched to just .dirmeta for metadata file names.
|
||||
metaName = ""
|
||||
}
|
||||
|
||||
if name == "root:" {
|
||||
return c
|
||||
}
|
||||
|
||||
md := onedriveMetadata(
|
||||
name,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
metaName+metadata.DirMetaFileSuffix,
|
||||
perm,
|
||||
true)
|
||||
|
||||
c.items = append(c.items, md)
|
||||
c.aux = append(c.aux, md)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
type oneDriveData struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Data []byte `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func onedriveItemWithData(
|
||||
name, lookupKey string,
|
||||
fileData []byte,
|
||||
) itemInfo {
|
||||
content := oneDriveData{
|
||||
FileName: lookupKey,
|
||||
Data: fileData,
|
||||
}
|
||||
|
||||
serialized, _ := json.Marshal(content)
|
||||
|
||||
return itemInfo{
|
||||
name: name,
|
||||
data: serialized,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
func onedriveMetadata(
|
||||
fileName, itemID, lookupKey string,
|
||||
perm permData,
|
||||
permUseID bool,
|
||||
) itemInfo {
|
||||
meta := getMetadata(fileName, perm, permUseID)
|
||||
|
||||
metaJSON, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
fmt.Println("marshalling metadata", clues.ToCore(err))
|
||||
}
|
||||
|
||||
return itemInfo{
|
||||
name: itemID,
|
||||
data: metaJSON,
|
||||
lookupKey: lookupKey,
|
||||
}
|
||||
}
|
||||
|
||||
func getMetadata(fileName string, perm permData, permUseID bool) onedrive.Metadata {
|
||||
if len(perm.user) == 0 || len(perm.roles) == 0 ||
|
||||
perm.sharingMode != onedrive.SharingModeCustom {
|
||||
return onedrive.Metadata{
|
||||
FileName: fileName,
|
||||
SharingMode: perm.sharingMode,
|
||||
}
|
||||
}
|
||||
|
||||
// In case of permissions, the id will usually be same for same
|
||||
// user/role combo unless deleted and readded, but we have to do
|
||||
// this as we only have two users of which one is already taken.
|
||||
id := uuid.NewString()
|
||||
uperm := onedrive.UserPermission{ID: id, Roles: perm.roles}
|
||||
|
||||
if permUseID {
|
||||
uperm.EntityID = perm.entityID
|
||||
} else {
|
||||
uperm.Email = perm.user
|
||||
}
|
||||
|
||||
meta := onedrive.Metadata{
|
||||
FileName: fileName,
|
||||
Permissions: []onedrive.UserPermission{uperm},
|
||||
}
|
||||
|
||||
return meta
|
||||
}
|
||||
|
||||
@ -51,7 +51,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
gc, acct, err := getGCAndVerifyUser(ctx, User)
|
||||
gc, acct, _, err := getGCAndVerifyUser(ctx, User)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
@ -98,7 +98,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
|
||||
return nil
|
||||
}
|
||||
|
||||
gc, acct, err := getGCAndVerifyUser(ctx, User)
|
||||
gc, acct, _, err := getGCAndVerifyUser(ctx, User)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
@ -144,7 +144,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
gc, acct, err := getGCAndVerifyUser(ctx, User)
|
||||
gc, acct, _, err := getGCAndVerifyUser(ctx, User)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
package impl
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
var filesCmd = &cobra.Command{
|
||||
@ -18,11 +24,44 @@ func AddOneDriveCommands(cmd *cobra.Command) {
|
||||
}
|
||||
|
||||
func handleOneDriveFileFactory(cmd *cobra.Command, args []string) error {
|
||||
Err(cmd.Context(), ErrNotYetImplemented)
|
||||
var (
|
||||
ctx = cmd.Context()
|
||||
service = path.OneDriveService
|
||||
category = path.FilesCategory
|
||||
errs = fault.New(false)
|
||||
)
|
||||
|
||||
if utils.HasNoFlagsAndShownHelp(cmd) {
|
||||
return nil
|
||||
}
|
||||
|
||||
gc, acct, inp, err := getGCAndVerifyUser(ctx, User)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
deets, err := generateAndRestoreOnedriveItems(
|
||||
gc,
|
||||
User,
|
||||
inp.ID(),
|
||||
strings.ToLower(SecondaryUser),
|
||||
acct,
|
||||
service,
|
||||
category,
|
||||
selectors.NewOneDriveBackup([]string{User}).Selector,
|
||||
Tenant,
|
||||
Destination,
|
||||
Count,
|
||||
errs)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
for _, e := range errs.Recovered() {
|
||||
logger.CtxErr(ctx, err).Error(e.Error())
|
||||
}
|
||||
|
||||
deets.PrintEntries(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -83,7 +83,7 @@ func main() {
|
||||
case "exchange":
|
||||
checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
|
||||
case "onedrive":
|
||||
checkOnedriveRestoration(ctx, client, testUser, folder, startTime)
|
||||
checkOnedriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
|
||||
default:
|
||||
fatal(ctx, "no service specified", nil)
|
||||
}
|
||||
@ -296,7 +296,7 @@ func checkOnedriveRestoration(
|
||||
ctx context.Context,
|
||||
client *msgraphsdk.GraphServiceClient,
|
||||
testUser,
|
||||
folderName string,
|
||||
folderName, dataFolder string,
|
||||
startTime time.Time,
|
||||
) {
|
||||
var (
|
||||
@ -337,7 +337,6 @@ func checkOnedriveRestoration(
|
||||
var (
|
||||
itemID = ptr.Val(driveItem.GetId())
|
||||
itemName = ptr.Val(driveItem.GetName())
|
||||
ictx = clues.Add(ctx, "item_id", itemID, "item_name", itemName)
|
||||
)
|
||||
|
||||
if itemName == folderName {
|
||||
@ -345,8 +344,10 @@ func checkOnedriveRestoration(
|
||||
continue
|
||||
}
|
||||
|
||||
folderTime, hasTime := mustGetTimeFromName(ictx, itemName)
|
||||
if !isWithinTimeBound(ctx, startTime, folderTime, hasTime) {
|
||||
if itemName != dataFolder {
|
||||
logger.Ctx(ctx).Infof("test data for %v folder: ", dataFolder)
|
||||
fmt.Printf("test data for %v folder: ", dataFolder)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@ -375,8 +376,7 @@ func checkOnedriveRestoration(
|
||||
getRestoredDrive(ctx, client, *drive.GetId(), restoreFolderID, restoreFile, restoreFolderPermission, startTime)
|
||||
|
||||
for folderName, permissions := range folderPermission {
|
||||
logger.Ctx(ctx).Info("checking for folder: ", folderName)
|
||||
fmt.Printf("checking for folder: %s\n", folderName)
|
||||
logAndPrint(ctx, "checking for folder: %s", folderName)
|
||||
|
||||
restoreFolderPerm := restoreFolderPermission[folderName]
|
||||
|
||||
@ -415,6 +415,9 @@ func checkOnedriveRestoration(
|
||||
}
|
||||
|
||||
for fileName, expected := range fileSizes {
|
||||
logger.Ctx(ctx).Info("checking for file: ", fileName)
|
||||
fmt.Printf("checking for file: %s\n", fileName)
|
||||
|
||||
got := restoreFile[fileName]
|
||||
|
||||
assert(
|
||||
@ -465,8 +468,7 @@ func getOneDriveChildFolder(
|
||||
// currently we don't restore blank folders.
|
||||
// skip permission check for empty folders
|
||||
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
||||
logger.Ctx(ctx).Info("skipped empty folder: ", fullName)
|
||||
fmt.Println("skipped empty folder: ", fullName)
|
||||
logAndPrint(ctx, "skipped empty folder: %s", fullName)
|
||||
|
||||
continue
|
||||
}
|
||||
@ -633,3 +635,8 @@ func assert(
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func logAndPrint(ctx context.Context, tmpl string, vs ...any) {
|
||||
logger.Ctx(ctx).Infof(tmpl, vs...)
|
||||
fmt.Printf(tmpl+"\n", vs...)
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user