larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use ids for CollectionsMap for OneDrive (#2457)

Browse Source 
## Description

This will help us with delta backups where we might run into a single
folder changing multiple times.

## Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/pull/2407

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2023-02-10 23:27:10 +05:30 committed by GitHub
parent b3a1de89bb
commit 66e5e7693d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 251 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/internal/connector/onedrive/collections.go
View File

@ -28,7 +28,11 @@ const (
OneDriveSource OneDriveSource
SharePointSource SharePointSource
) )
const restrictedDirectory = "Site Pages"
const (
restrictedDirectory = "Site Pages"
rootDrivePattern = "/drives/%s/root:"
)
func (ds driveSource) toPathServiceCat() (path.ServiceType, path.CategoryType) { func (ds driveSource) toPathServiceCat() (path.ServiceType, path.CategoryType) {
switch ds { switch ds {
@ -382,11 +386,15 @@ func (c *Collections) UpdateCollections(
continue continue
} }
if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil { if item.GetParentReference() == nil ||
item.GetParentReference().GetPath() == nil ||
item.GetParentReference().GetId() == nil {
return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName()) return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName())
} }
// Create a collection for the parent of this item // Create a collection for the parent of this item
collectionID := *item.GetParentReference().GetId()
collectionPath, err := GetCanonicalPath( collectionPath, err := GetCanonicalPath(
*item.GetParentReference().GetPath(), *item.GetParentReference().GetPath(),
c.tenant, c.tenant,
@ -454,8 +462,7 @@ func (c *Collections) UpdateCollections(
// TODO(ashmrtn): Figure what when an item was moved (maybe) and add it to // TODO(ashmrtn): Figure what when an item was moved (maybe) and add it to
// the exclude list. // the exclude list.
col, found := c.CollectionMap[collectionPath.String()] col, found := c.CollectionMap[collectionID]
if !found { if !found {
// TODO(ashmrtn): Compare old and new path and set collection state // TODO(ashmrtn): Compare old and new path and set collection state
// accordingly. // accordingly.
@ -470,7 +477,7 @@ func (c *Collections) UpdateCollections(
invalidPrevDelta, invalidPrevDelta,
) )
c.CollectionMap[collectionPath.String()] = col c.CollectionMap[collectionID] = col
c.NumContainers++ c.NumContainers++
} }

369
src/internal/connector/onedrive/collections_test.go
View File

@ -2,6 +2,7 @@ package onedrive
import ( import (
"context" "context"
"fmt"
"strings" "strings"
"testing" "testing"
@ -23,10 +24,6 @@ import (
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
) )
const (
testBaseDrivePath = "drive/driveID1/root:"
)
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string { func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {
res := make([]string, 0, len(rest)) res := make([]string, 0, len(rest))
@ -102,12 +99,15 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
pkg = "/package" pkg = "/package"
) )
testBaseDrivePath := fmt.Sprintf(rootDrivePattern, "driveID1")
tests := []struct { tests := []struct {
testCase string testCase string
items []models.DriveItemable items []models.DriveItemable
inputFolderMap map[string]string inputFolderMap map[string]string
scope selectors.OneDriveScope scope selectors.OneDriveScope
expect assert.ErrorAssertionFunc expect assert.ErrorAssertionFunc
expectedCollectionIDs []string
expectedCollectionPaths []string expectedCollectionPaths []string
expectedItemCount int expectedItemCount int
expectedContainerCount int expectedContainerCount int
@ -118,7 +118,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "Invalid item", testCase: "Invalid item",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("item", "item", testBaseDrivePath, false, false, false), driveRootItem("root"),
driveItem("item", "item", testBaseDrivePath, "root", false, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
@ -129,11 +130,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "Single File", testCase: "Single File",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath, "root", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -150,11 +153,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "Single Folder", testCase: "Single Folder",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -176,11 +181,13 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "Single Package", testCase: "Single Package",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("package", "package", testBaseDrivePath, false, false, true), driveRootItem("root"),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -202,15 +209,17 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "1 root file, 1 folder, 1 package, 2 files, 3 collections", testCase: "1 root file, 1 folder, 1 package, 2 files, 3 collections",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root", "folder", "package"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -241,18 +250,20 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "contains folder selector", testCase: "contains folder selector",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false),
driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"folder", "subfolder", "folder2"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -285,19 +296,21 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "prefix subfolder selector", testCase: "prefix subfolder selector",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folderSub, false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}). scope: (&selectors.OneDriveBackup{}).
Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0], Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"subfolder", "folder2"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -321,17 +334,19 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "match subfolder selector", testCase: "match subfolder selector",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, false, true, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("package", "package", testBaseDrivePath, false, false, true), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, true, false, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, true, false, false), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, true, false, false), driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, "subfolder", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"subfolder"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -348,7 +363,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "not moved folder tree", testCase: "not moved folder tree",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -364,8 +380,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
testBaseDrivePath+"/folder/subfolder", testBaseDrivePath+"/folder/subfolder",
)[0], )[0],
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -394,7 +411,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "moved folder tree", testCase: "moved folder tree",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -410,8 +428,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
testBaseDrivePath+"/a-folder/subfolder", testBaseDrivePath+"/a-folder/subfolder",
)[0], )[0],
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -440,8 +459,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "moved folder tree and subfolder 1", testCase: "moved folder tree and subfolder 1",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -457,8 +477,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
testBaseDrivePath+"/a-folder/subfolder", testBaseDrivePath+"/a-folder/subfolder",
)[0], )[0],
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -487,8 +508,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "moved folder tree and subfolder 2", testCase: "moved folder tree and subfolder 2",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath, "root", false, true, false),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -504,8 +526,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
testBaseDrivePath+"/a-folder/subfolder", testBaseDrivePath+"/a-folder/subfolder",
)[0], )[0],
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -534,8 +557,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "deleted folder and package", testCase: "deleted folder and package",
items: []models.DriveItemable{ items: []models.DriveItemable{
delItem("folder", testBaseDrivePath, false, true, false), driveRootItem("root"), // root is always present, but not necessary here
delItem("package", testBaseDrivePath, false, false, true), delItem("folder", testBaseDrivePath, "root", false, true, false),
delItem("package", testBaseDrivePath, "root", false, false, true),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -563,8 +587,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "delete folder tree move subfolder", testCase: "delete folder tree move subfolder",
items: []models.DriveItemable{ items: []models.DriveItemable{
delItem("folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("subfolder", "subfolder", testBaseDrivePath, false, true, false), delItem("folder", testBaseDrivePath, "root", false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath, "root", false, true, false),
}, },
inputFolderMap: map[string]string{ inputFolderMap: map[string]string{
"folder": expectedPathAsSlice( "folder": expectedPathAsSlice(
@ -580,8 +605,9 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
testBaseDrivePath+"/folder/subfolder", testBaseDrivePath+"/folder/subfolder",
)[0], )[0],
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -604,7 +630,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
{ {
testCase: "delete file", testCase: "delete file",
items: []models.DriveItemable{ items: []models.DriveItemable{
delItem("item", testBaseDrivePath, true, false, false), delItem("item", testBaseDrivePath, "root", true, false, false),
}, },
inputFolderMap: map[string]string{}, inputFolderMap: map[string]string{},
scope: anyFolder, scope: anyFolder,
@ -640,7 +666,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
err := c.UpdateCollections( err := c.UpdateCollections(
ctx, ctx,
"driveID", "driveID1",
"General", "General",
tt.items, tt.items,
tt.inputFolderMap, tt.inputFolderMap,
@ -649,14 +675,19 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
false, false,
) )
tt.expect(t, err) tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, len(tt.expectedCollectionIDs), len(c.CollectionMap), "collection ids")
assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, tt.expectedFileCount, c.NumFiles, "file count") assert.Equal(t, tt.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, tt.expectedContainerCount, c.NumContainers, "container count") assert.Equal(t, tt.expectedContainerCount, c.NumContainers, "container count")
for _, collPath := range tt.expectedCollectionPaths {
for _, collPath := range tt.expectedCollectionIDs {
assert.Contains(t, c.CollectionMap, collPath) assert.Contains(t, c.CollectionMap, collPath)
} }
for _, col := range c.CollectionMap {
assert.Contains(t, tt.expectedCollectionPaths, col.FullPath().String())
}
assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap) assert.Equal(t, tt.expectedMetadataPaths, outputFolderMap)
assert.Equal(t, tt.expectedExcludes, excludes) assert.Equal(t, tt.expectedExcludes, excludes)
}) })
@ -1080,19 +1111,6 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
) )
require.NoError(suite.T(), err, "making metadata path") require.NoError(suite.T(), err, "making metadata path")
rootFolderPath := expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath,
)[0]
folderPath := expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
)[0]
empty := "" empty := ""
next := "next" next := "next"
delta := "delta1" delta := "delta1"
@ -1108,7 +1126,21 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
drive2.SetId(&driveID2) drive2.SetId(&driveID2)
drive2.SetName(&driveID2) drive2.SetName(&driveID2)
driveBasePath2 := "drive/driveID2/root:" driveBasePath1 := fmt.Sprintf(rootDrivePattern, driveID1)
driveBasePath2 := fmt.Sprintf(rootDrivePattern, driveID2)
rootFolderPath1 := expectedPathAsSlice(
suite.T(),
tenant,
user,
driveBasePath1,
)[0]
folderPath1 := expectedPathAsSlice(
suite.T(),
tenant,
user,
driveBasePath1+"/folder",
)[0]
rootFolderPath2 := expectedPathAsSlice( rootFolderPath2 := expectedPathAsSlice(
suite.T(), suite.T(),
@ -1130,7 +1162,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
errCheck assert.ErrorAssertionFunc errCheck assert.ErrorAssertionFunc
// Collection name -> set of item IDs. We can't check item data because // Collection name -> set of item IDs. We can't check item data because
// that's not mocked out. Metadata is checked separately. // that's not mocked out. Metadata is checked separately.
expectedCollections map[string][]string expectedCollections map[string]map[data.CollectionState][]string
expectedDeltaURLs map[string]string expectedDeltaURLs map[string]string
expectedFolderPaths map[string]map[string]string expectedFolderPaths map[string]map[string]string
expectedDelList map[string]struct{} expectedDelList map[string]struct{}
@ -1143,14 +1175,15 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
delItem("file", testBaseDrivePath, true, false, false), driveRootItem("root"), // will be present, not needed
delItem("file", driveBasePath1, "root", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{}, expectedCollections: map[string]map[data.CollectionState][]string{},
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
}, },
@ -1170,20 +1203,21 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath, driveBasePath1,
)[0]: {"file"}, )[0]: {data.NewState: {"file"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
@ -1202,24 +1236,25 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
folderPath: {"file"}, folderPath1: {data.NewState: {"file"}},
rootFolderPath: {"folder"}, rootFolderPath1: {data.NewState: {"folder"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
}, },
expectedFolderPaths: map[string]map[string]string{ expectedFolderPaths: map[string]map[string]string{
driveID1: { driveID1: {
"folder": folderPath, "folder": folderPath1,
}, },
}, },
expectedDelList: map[string]struct{}{}, expectedDelList: map[string]struct{}{},
@ -1231,17 +1266,18 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &empty, deltaLink: &empty,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
folderPath: {"file"}, folderPath1: {data.NewState: {"file"}},
rootFolderPath: {"folder"}, rootFolderPath1: {data.NewState: {"folder"}},
}, },
expectedDeltaURLs: map[string]string{}, expectedDeltaURLs: map[string]string{},
expectedFolderPaths: map[string]map[string]string{}, expectedFolderPaths: map[string]map[string]string{},
@ -1254,31 +1290,33 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
nextLink: &next, nextLink: &next,
}, },
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("file2", "file2", testBaseDrivePath+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
folderPath: {"file", "file2"}, folderPath1: {data.NewState: {"file", "file2"}},
rootFolderPath: {"folder"}, rootFolderPath1: {data.NewState: {"folder"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
}, },
expectedFolderPaths: map[string]map[string]string{ expectedFolderPaths: map[string]map[string]string{
driveID1: { driveID1: {
"folder": folderPath, "folder": folderPath1,
}, },
}, },
expectedDelList: map[string]struct{}{}, expectedDelList: map[string]struct{}{},
@ -1293,8 +1331,9 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", testBaseDrivePath, false, true, false), driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
@ -1302,19 +1341,20 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
driveID2: { driveID2: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("folder", "folder", driveBasePath2, false, true, false), driveRootItem("root"),
driveItem("file", "file", driveBasePath2+"/folder", true, false, false), driveItem("folder", "folder", driveBasePath2, "root", false, true, false),
driveItem("file", "file", driveBasePath2+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta2, deltaLink: &delta2,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
folderPath: {"file"}, folderPath1: {data.NewState: {"file"}},
folderPath2: {"file"}, folderPath2: {data.NewState: {"file"}},
rootFolderPath: {"folder"}, rootFolderPath1: {data.NewState: {"folder"}},
rootFolderPath2: {"folder"}, rootFolderPath2: {data.NewState: {"folder"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
@ -1322,7 +1362,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
}, },
expectedFolderPaths: map[string]map[string]string{ expectedFolderPaths: map[string]map[string]string{
driveID1: { driveID1: {
"folder": folderPath, "folder": folderPath1,
}, },
driveID2: { driveID2: {
"folder": folderPath2, "folder": folderPath2,
@ -1356,20 +1396,21 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
}, },
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath, driveBasePath1,
)[0]: {"file"}, )[0]: {data.NewState: {"file"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
@ -1383,7 +1424,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
doNotMergeItems: true, doNotMergeItems: true,
}, },
{ {
name: "OneDrive_MultipleCollections_DeltaError", name: "OneDrive_TwoItemPage_DeltaError",
drives: []models.Driveable{drive1}, drives: []models.Driveable{drive1},
items: map[string][]deltaPagerResult{ items: map[string][]deltaPagerResult{
driveID1: { driveID1: {
@ -1392,85 +1433,87 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
}, },
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
}, },
nextLink: &next, nextLink: &next,
}, },
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath, driveBasePath1,
)[0]: {"file"}, )[0]: {data.NewState: {"file", "folder"}},
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath+"/folder", driveBasePath1+"/folder",
)[0]: {"file"}, )[0]: {data.NewState: {"file"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
}, },
expectedFolderPaths: map[string]map[string]string{ expectedFolderPaths: map[string]map[string]string{
// We need an empty map here so deserializing metadata knows the delta driveID1: {"folder": folderPath1},
// token for this drive is valid.
driveID1: {},
}, },
expectedDelList: map[string]struct{}{}, expectedDelList: map[string]struct{}{},
doNotMergeItems: true, doNotMergeItems: true,
}, },
{ {
name: "OneDrive_MultipleCollections_NoDeltaError", name: "OneDrive_TwoItemPage_NoDeltaError",
drives: []models.Driveable{drive1}, drives: []models.Driveable{drive1},
items: map[string][]deltaPagerResult{ items: map[string][]deltaPagerResult{
driveID1: { driveID1: {
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath, true, false, false), driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
}, },
nextLink: &next, nextLink: &next,
}, },
{ {
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", "file", testBaseDrivePath+"/folder", true, false, false), driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}, },
deltaLink: &delta, deltaLink: &delta,
}, },
}, },
}, },
errCheck: assert.NoError, errCheck: assert.NoError,
expectedCollections: map[string][]string{ expectedCollections: map[string]map[data.CollectionState][]string{
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath, driveBasePath1,
)[0]: {"file"}, )[0]: {data.NewState: {"file", "folder"}},
expectedPathAsSlice( expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
user, user,
testBaseDrivePath+"/folder", driveBasePath1+"/folder",
)[0]: {"file"}, )[0]: {data.NewState: {"file"}},
}, },
expectedDeltaURLs: map[string]string{ expectedDeltaURLs: map[string]string{
driveID1: delta, driveID1: delta,
}, },
expectedFolderPaths: map[string]map[string]string{ expectedFolderPaths: map[string]map[string]string{
// We need an empty map here so deserializing metadata knows the delta driveID1: {"folder": folderPath1},
// token for this drive is valid.
driveID1: {},
}, },
expectedDelList: map[string]struct{}{}, expectedDelList: map[string]struct{}{},
doNotMergeItems: false, doNotMergeItems: false,
@ -1555,7 +1598,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
itemIDs = append(itemIDs, id) itemIDs = append(itemIDs, id)
} }
assert.ElementsMatch(t, test.expectedCollections[folderPath], itemIDs) assert.ElementsMatch(t, test.expectedCollections[folderPath][baseCol.State()], itemIDs)
assert.Equal(t, test.doNotMergeItems, baseCol.DoNotMergeItems(), "DoNotMergeItems") assert.Equal(t, test.doNotMergeItems, baseCol.DoNotMergeItems(), "DoNotMergeItems")
} }
@ -1568,6 +1611,7 @@ func driveItem(
id string, id string,
name string, name string,
parentPath string, parentPath string,
parentID string,
isFile, isFolder, isPackage bool, isFile, isFolder, isPackage bool,
) models.DriveItemable { ) models.DriveItemable {
item := models.NewDriveItem() item := models.NewDriveItem()
@ -1576,6 +1620,7 @@ func driveItem(
parentReference := models.NewItemReference() parentReference := models.NewItemReference()
parentReference.SetPath(&parentPath) parentReference.SetPath(&parentPath)
parentReference.SetId(&parentID)
item.SetParentReference(parentReference) item.SetParentReference(parentReference)
switch { switch {
@ -1590,11 +1635,22 @@ func driveItem(
return item return item
} }
func driveRootItem(id string) models.DriveItemable {
name := "root"
item := models.NewDriveItem()
item.SetName(&name)
item.SetId(&id)
item.SetRoot(models.NewRoot())
return item
}
// delItem creates a DriveItemable that is marked as deleted. path must be set // delItem creates a DriveItemable that is marked as deleted. path must be set
// to the base drive path. // to the base drive path.
func delItem( func delItem(
id string, id string,
parentPath string, parentPath string,
parentID string,
isFile, isFolder, isPackage bool, isFile, isFolder, isPackage bool,
) models.DriveItemable { ) models.DriveItemable {
item := models.NewDriveItem() item := models.NewDriveItem()
@ -1603,6 +1659,7 @@ func delItem(
parentReference := models.NewItemReference() parentReference := models.NewItemReference()
parentReference.SetPath(&parentPath) parentReference.SetPath(&parentPath)
parentReference.SetId(&parentID)
item.SetParentReference(parentReference) item.SetParentReference(parentReference)
switch { switch {

35
src/internal/connector/sharepoint/data_collections_test.go
View File

@ -20,7 +20,7 @@ import (
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const ( const (
testBaseDrivePath = "drive/driveID1/root:" testBaseDrivePath = "drives/driveID1/root:"
) )
type testFolderMatcher struct { type testFolderMatcher struct {
@ -60,6 +60,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
items []models.DriveItemable items []models.DriveItemable
scope selectors.SharePointScope scope selectors.SharePointScope
expect assert.ErrorAssertionFunc expect assert.ErrorAssertionFunc
expectedCollectionIDs []string
expectedCollectionPaths []string expectedCollectionPaths []string
expectedItemCount int expectedItemCount int
expectedContainerCount int expectedContainerCount int
@ -68,10 +69,12 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
{ {
testCase: "Single File", testCase: "Single File",
items: []models.DriveItemable{ items: []models.DriveItemable{
driveItem("file", testBaseDrivePath, true), driveRootItem("root"),
driveItem("file", testBaseDrivePath, "root", true),
}, },
scope: anyFolder, scope: anyFolder,
expect: assert.NoError, expect: assert.NoError,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice( expectedCollectionPaths: expectedPathAsSlice(
suite.T(), suite.T(),
tenant, tenant,
@ -101,26 +104,30 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
&MockGraphService{}, &MockGraphService{},
nil, nil,
control.Options{}) control.Options{})
err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths, newPaths, excluded, true) err := c.UpdateCollections(ctx, "driveID1", "General", test.items, paths, newPaths, excluded, true)
test.expect(t, err) test.expect(t, err)
assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, len(test.expectedCollectionIDs), len(c.CollectionMap), "collection paths")
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count") assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count") assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
for _, collPath := range test.expectedCollectionPaths { for _, collPath := range test.expectedCollectionIDs {
assert.Contains(t, c.CollectionMap, collPath) assert.Contains(t, c.CollectionMap, collPath)
} }
for _, col := range c.CollectionMap {
assert.Contains(t, test.expectedCollectionPaths, col.FullPath().String())
}
}) })
} }
} }
func driveItem(name string, path string, isFile bool) models.DriveItemable { func driveItem(name, parentPath, parentID string, isFile bool) models.DriveItemable {
item := models.NewDriveItem() item := models.NewDriveItem()
item.SetName(&name) item.SetName(&name)
item.SetId(&name) item.SetId(&name)
parentReference := models.NewItemReference() parentReference := models.NewItemReference()
parentReference.SetPath(&path) parentReference.SetPath(&parentPath)
parentReference.SetId(&parentID)
item.SetParentReference(parentReference) item.SetParentReference(parentReference)
if isFile { if isFile {
@ -130,6 +137,16 @@ func driveItem(name string, path string, isFile bool) models.DriveItemable {
return item return item
} }
func driveRootItem(id string) models.DriveItemable {
name := "root"
item := models.NewDriveItem()
item.SetName(&name)
item.SetId(&id)
item.SetRoot(models.NewRoot())
return item
}
type SharePointPagesSuite struct { type SharePointPagesSuite struct {
suite.Suite suite.Suite
} }