Centralize metadata suffix definitions (#5134)

<!-- PR description-->

Centralizing the suffixes so that group mailbox and future services can also use them.

* Move `.meta`, `.data`, `.dirmeta` definitions from `internal/m365/collection/drive/metadata` pkg to `pkg/services/m365/api/graph/metadata`.
* Adjust package names in misc places.
* No logic change. 
* We should probably keep all metadata definitions and code in one place, i.e. relocate drive metadata definitions to `pkg/services/m365/api/graph/metadata`. This is a bigger change, we can do it later.
---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2024-01-26 14:06:04 -08:00 committed by GitHub
parent 683f163df1
commit 8133da3087
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 96 additions and 93 deletions

View File

@ -27,7 +27,6 @@ import (
strTD "github.com/alcionai/corso/src/internal/common/str/testdata" strTD "github.com/alcionai/corso/src/internal/common/str/testdata"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
istats "github.com/alcionai/corso/src/internal/stats" istats "github.com/alcionai/corso/src/internal/stats"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -38,6 +37,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
) )

View File

@ -16,7 +16,6 @@ import (
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
@ -29,6 +28,7 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"
) )

View File

@ -21,7 +21,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/readers" "github.com/alcionai/corso/src/internal/common/readers"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
metaTD "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata/testdata" metaTD "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata/testdata"
odTD "github.com/alcionai/corso/src/internal/m365/service/onedrive/testdata" odTD "github.com/alcionai/corso/src/internal/m365/service/onedrive/testdata"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
@ -34,6 +34,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"
) )
@ -73,13 +74,13 @@ func (suite *CollectionUnitSuite) TestCollection() {
stubMetaID = "testMetaID" stubMetaID = "testMetaID"
stubMetaEntityID = "email@provider.com" stubMetaEntityID = "email@provider.com"
stubMetaRoles = []string{"read", "write"} stubMetaRoles = []string{"read", "write"}
stubMeta = metadata.Metadata{ stubMeta = odmetadata.Metadata{
FileName: stubItemName, FileName: stubItemName,
Permissions: []metadata.Permission{ Permissions: []odmetadata.Permission{
{ {
ID: stubMetaID, ID: stubMetaID,
EntityID: stubMetaEntityID, EntityID: stubMetaEntityID,
EntityType: metadata.GV2User, EntityType: odmetadata.GV2User,
Roles: stubMetaRoles, Roles: stubMetaRoles,
Expiration: &now, Expiration: &now,
}, },
@ -208,7 +209,7 @@ func (suite *CollectionUnitSuite) TestCollection() {
mbh.GetErrs = []error{test.getErr} mbh.GetErrs = []error{test.getErr}
mbh.GI = getsItem{Err: assert.AnError} mbh.GI = getsItem{Err: assert.AnError}
pcr := metaTD.NewStubPermissionResponse(metadata.GV2User, stubMetaID, stubMetaEntityID, stubMetaRoles) pcr := metaTD.NewStubPermissionResponse(odmetadata.GV2User, stubMetaID, stubMetaEntityID, stubMetaRoles)
mbh.GIP = getsItemPermission{Perm: pcr} mbh.GIP = getsItemPermission{Perm: pcr}
coll, err := NewCollection( coll, err := NewCollection(
@ -294,7 +295,7 @@ func (suite *CollectionUnitSuite) TestCollection() {
assert.Equal(t, readers.DefaultSerializationVersion, rr.Format().Version) assert.Equal(t, readers.DefaultSerializationVersion, rr.Format().Version)
assert.False(t, rr.Format().DelInFlight) assert.False(t, rr.Format().DelInFlight)
readMeta := metadata.Metadata{} readMeta := odmetadata.Metadata{}
err = json.NewDecoder(rr).Decode(&readMeta) err = json.NewDecoder(rr).Decode(&readMeta)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))

View File

@ -14,7 +14,6 @@ import (
"github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/prefixmatcher"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
bupMD "github.com/alcionai/corso/src/pkg/backup/metadata" bupMD "github.com/alcionai/corso/src/pkg/backup/metadata"
@ -26,6 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"
) )

View File

@ -9,9 +9,9 @@ import (
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"
) )

View File

@ -7,13 +7,13 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/metrics" "github.com/alcionai/corso/src/pkg/metrics"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
func NewExportCollection( func NewExportCollection(

View File

@ -12,9 +12,9 @@ import (
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
type ExportUnitSuite struct { type ExportUnitSuite struct {

View File

@ -19,7 +19,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -34,6 +33,7 @@ import (
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"

View File

@ -10,12 +10,13 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/syncd" "github.com/alcionai/corso/src/internal/common/syncd"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
// empty string is used to indicate that a permission cannot be restored // empty string is used to indicate that a permission cannot be restored
@ -23,20 +24,20 @@ const nonRestorablePermission = ""
func getParentMetadata( func getParentMetadata(
parentPath path.Path, parentPath path.Path,
parentDirToMeta syncd.MapTo[metadata.Metadata], parentDirToMeta syncd.MapTo[odmetadata.Metadata],
) (metadata.Metadata, error) { ) (odmetadata.Metadata, error) {
parentMeta, ok := parentDirToMeta.Load(parentPath.String()) parentMeta, ok := parentDirToMeta.Load(parentPath.String())
if !ok { if !ok {
drivePath, err := path.ToDrivePath(parentPath) drivePath, err := path.ToDrivePath(parentPath)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.Wrap(err, "invalid restore path") return odmetadata.Metadata{}, clues.Wrap(err, "invalid restore path")
} }
if len(drivePath.Folders) != 0 { if len(drivePath.Folders) != 0 {
return metadata.Metadata{}, clues.Wrap(err, "computing item permissions") return odmetadata.Metadata{}, clues.Wrap(err, "computing item permissions")
} }
parentMeta = metadata.Metadata{} parentMeta = odmetadata.Metadata{}
} }
return parentMeta, nil return parentMeta, nil
@ -49,9 +50,9 @@ func getCollectionMetadata(
caches *restoreCaches, caches *restoreCaches,
backupVersion int, backupVersion int,
restorePerms bool, restorePerms bool,
) (metadata.Metadata, error) { ) (odmetadata.Metadata, error) {
if !restorePerms || backupVersion < version.OneDrive1DataAndMetaFiles { if !restorePerms || backupVersion < version.OneDrive1DataAndMetaFiles {
return metadata.Metadata{}, nil return odmetadata.Metadata{}, nil
} }
var ( var (
@ -61,13 +62,13 @@ func getCollectionMetadata(
if len(drivePath.Folders) == 0 { if len(drivePath.Folders) == 0 {
// No permissions for root folder // No permissions for root folder
return metadata.Metadata{}, nil return odmetadata.Metadata{}, nil
} }
if backupVersion < version.OneDrive4DirIncludesPermissions { if backupVersion < version.OneDrive4DirIncludesPermissions {
colMeta, err := getParentMetadata(fullPath, caches.ParentDirToMeta) colMeta, err := getParentMetadata(fullPath, caches.ParentDirToMeta)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.Wrap(err, "collection metadata") return odmetadata.Metadata{}, clues.Wrap(err, "collection metadata")
} }
return colMeta, nil return colMeta, nil
@ -82,7 +83,7 @@ func getCollectionMetadata(
meta, err := FetchAndReadMetadata(ctx, dc, metaName) meta, err := FetchAndReadMetadata(ctx, dc, metaName)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.Wrap(err, "collection metadata") return odmetadata.Metadata{}, clues.Wrap(err, "collection metadata")
} }
return meta, nil return meta, nil
@ -93,9 +94,9 @@ func getCollectionMetadata(
func computePreviousLinkShares( func computePreviousLinkShares(
ctx context.Context, ctx context.Context,
originDir path.Path, originDir path.Path,
parentMetas syncd.MapTo[metadata.Metadata], parentMetas syncd.MapTo[odmetadata.Metadata],
) ([]metadata.LinkShare, error) { ) ([]odmetadata.LinkShare, error) {
linkShares := []metadata.LinkShare{} linkShares := []odmetadata.LinkShare{}
ctx = clues.Add(ctx, "origin_dir", originDir) ctx = clues.Add(ctx, "origin_dir", originDir)
parent, err := originDir.Dir() parent, err := originDir.Dir()
@ -122,7 +123,7 @@ func computePreviousLinkShares(
// Any change in permissions would change it to custom // Any change in permissions would change it to custom
// permission set and so we can filter on that. // permission set and so we can filter on that.
if meta.SharingMode == metadata.SharingModeCustom { if meta.SharingMode == odmetadata.SharingModeCustom {
linkShares = append(linkShares, meta.LinkShares...) linkShares = append(linkShares, meta.LinkShares...)
} }
@ -143,11 +144,11 @@ func computePreviousMetadata(
ctx context.Context, ctx context.Context,
originDir path.Path, originDir path.Path,
// map parent dir -> parent's metadata // map parent dir -> parent's metadata
parentMetas syncd.MapTo[metadata.Metadata], parentMetas syncd.MapTo[odmetadata.Metadata],
) (metadata.Metadata, error) { ) (odmetadata.Metadata, error) {
var ( var (
parent path.Path parent path.Path
meta metadata.Metadata meta odmetadata.Metadata
err error err error
ok bool ok bool
@ -158,26 +159,26 @@ func computePreviousMetadata(
for { for {
parent, err = parent.Dir() parent, err = parent.Dir()
if err != nil { if err != nil {
return metadata.Metadata{}, clues.WrapWC(ctx, err, "getting parent") return odmetadata.Metadata{}, clues.WrapWC(ctx, err, "getting parent")
} }
ictx := clues.Add(ctx, "parent_dir", parent) ictx := clues.Add(ctx, "parent_dir", parent)
drivePath, err := path.ToDrivePath(parent) drivePath, err := path.ToDrivePath(parent)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.WrapWC(ictx, err, "transforming dir to drivePath") return odmetadata.Metadata{}, clues.WrapWC(ictx, err, "transforming dir to drivePath")
} }
if len(drivePath.Folders) == 0 { if len(drivePath.Folders) == 0 {
return metadata.Metadata{}, nil return odmetadata.Metadata{}, nil
} }
meta, ok = parentMetas.Load(parent.String()) meta, ok = parentMetas.Load(parent.String())
if !ok { if !ok {
return metadata.Metadata{}, clues.NewWC(ictx, "no metadata found for parent folder: "+parent.String()) return odmetadata.Metadata{}, clues.NewWC(ictx, "no metadata found for parent folder: "+parent.String())
} }
if meta.SharingMode == metadata.SharingModeCustom { if meta.SharingMode == odmetadata.SharingModeCustom {
return meta, nil return meta, nil
} }
} }
@ -195,7 +196,7 @@ func UpdatePermissions(
udip updateDeleteItemPermissioner, udip updateDeleteItemPermissioner,
driveID string, driveID string,
itemID string, itemID string,
permAdded, permRemoved []metadata.Permission, permAdded, permRemoved []odmetadata.Permission,
oldPermIDToNewID syncd.MapTo[string], oldPermIDToNewID syncd.MapTo[string],
errs *fault.Bus, errs *fault.Bus,
) error { ) error {
@ -260,7 +261,7 @@ func UpdatePermissions(
// TODO: sitegroup support. Currently errors with "One or more users could not be resolved", // TODO: sitegroup support. Currently errors with "One or more users could not be resolved",
// likely due to the site group entityID consisting of a single integer (ex: 4) // likely due to the site group entityID consisting of a single integer (ex: 4)
if len(roles) == 0 || p.EntityType == metadata.GV2SiteGroup { if len(roles) == 0 || p.EntityType == odmetadata.GV2SiteGroup {
continue continue
} }
@ -315,7 +316,7 @@ func UpdateLinkShares(
upils updateDeleteItemLinkSharer, upils updateDeleteItemLinkSharer,
driveID string, driveID string,
itemID string, itemID string,
lsAdded, lsRemoved []metadata.LinkShare, lsAdded, lsRemoved []odmetadata.LinkShare,
oldLinkShareIDToNewID syncd.MapTo[string], oldLinkShareIDToNewID syncd.MapTo[string],
errs *fault.Bus, errs *fault.Bus,
) (bool, error) { ) (bool, error) {
@ -347,7 +348,7 @@ func UpdateLinkShares(
for _, iden := range ls.Entities { for _, iden := range ls.Entities {
// TODO: sitegroup support. Currently errors with "One or more users could not be resolved", // TODO: sitegroup support. Currently errors with "One or more users could not be resolved",
// likely due to the site group entityID consisting of a single integer (ex: 4) // likely due to the site group entityID consisting of a single integer (ex: 4)
if iden.EntityType == metadata.GV2SiteGroup { if iden.EntityType == odmetadata.GV2SiteGroup {
continue continue
} }
@ -457,11 +458,11 @@ func UpdateLinkShares(
func filterUnavailableEntitiesInLinkShare( func filterUnavailableEntitiesInLinkShare(
ctx context.Context, ctx context.Context,
linkShares []metadata.LinkShare, linkShares []odmetadata.LinkShare,
availableEntities ResourceIDNames, availableEntities ResourceIDNames,
oldLinkShareIDToNewID syncd.MapTo[string], oldLinkShareIDToNewID syncd.MapTo[string],
) []metadata.LinkShare { ) []odmetadata.LinkShare {
filtered := []metadata.LinkShare{} filtered := []odmetadata.LinkShare{}
if availableEntities.Users == nil || availableEntities.Groups == nil { if availableEntities.Users == nil || availableEntities.Groups == nil {
// This should not be happening unless we missed to fill in the caches // This should not be happening unless we missed to fill in the caches
@ -470,20 +471,20 @@ func filterUnavailableEntitiesInLinkShare(
} }
for _, p := range linkShares { for _, p := range linkShares {
entities := []metadata.Entity{} entities := []odmetadata.Entity{}
for _, e := range p.Entities { for _, e := range p.Entities {
available := false available := false
switch e.EntityType { switch e.EntityType {
case metadata.GV2User: case odmetadata.GV2User:
// Link shares with external users won't have IDs // Link shares with external users won't have IDs
if len(e.ID) == 0 && len(e.Email) > 0 { if len(e.ID) == 0 && len(e.Email) > 0 {
available = true available = true
} else { } else {
_, available = availableEntities.Users.NameOf(e.ID) _, available = availableEntities.Users.NameOf(e.ID)
} }
case metadata.GV2Group: case odmetadata.GV2Group:
_, available = availableEntities.Groups.NameOf(e.ID) _, available = availableEntities.Groups.NameOf(e.ID)
default: default:
// We only know about users and groups // We only know about users and groups
@ -513,26 +514,26 @@ func filterUnavailableEntitiesInLinkShare(
func filterUnavailableEntitiesInPermissions( func filterUnavailableEntitiesInPermissions(
ctx context.Context, ctx context.Context,
perms []metadata.Permission, perms []odmetadata.Permission,
availableEntities ResourceIDNames, availableEntities ResourceIDNames,
oldPermIDToNewID syncd.MapTo[string], oldPermIDToNewID syncd.MapTo[string],
) []metadata.Permission { ) []odmetadata.Permission {
if availableEntities.Users == nil || availableEntities.Groups == nil { if availableEntities.Users == nil || availableEntities.Groups == nil {
// This should not be happening unless we missed to fill in the caches // This should not be happening unless we missed to fill in the caches
logger.Ctx(ctx).Info("no available entities, not filtering link shares") logger.Ctx(ctx).Info("no available entities, not filtering link shares")
return perms return perms
} }
filtered := []metadata.Permission{} filtered := []odmetadata.Permission{}
for _, p := range perms { for _, p := range perms {
available := false available := false
switch p.EntityType { switch p.EntityType {
case metadata.GV2User: case odmetadata.GV2User:
_, ok := availableEntities.Users.NameOf(p.EntityID) _, ok := availableEntities.Users.NameOf(p.EntityID)
available = available || ok available = available || ok
case metadata.GV2Group: case odmetadata.GV2Group:
_, ok := availableEntities.Groups.NameOf(p.EntityID) _, ok := availableEntities.Groups.NameOf(p.EntityID)
available = available || ok available = available || ok
default: default:
@ -564,11 +565,11 @@ func RestorePermissions(
driveID string, driveID string,
itemID string, itemID string,
itemPath path.Path, itemPath path.Path,
current metadata.Metadata, current odmetadata.Metadata,
caches *restoreCaches, caches *restoreCaches,
errs *fault.Bus, errs *fault.Bus,
) { ) {
if current.SharingMode == metadata.SharingModeInherited { if current.SharingMode == odmetadata.SharingModeInherited {
return return
} }
@ -582,7 +583,7 @@ func RestorePermissions(
} }
if previousLinkShares != nil { if previousLinkShares != nil {
lsAdded, lsRemoved := metadata.DiffLinkShares(previousLinkShares, current.LinkShares) lsAdded, lsRemoved := odmetadata.DiffLinkShares(previousLinkShares, current.LinkShares)
lsAdded = filterUnavailableEntitiesInLinkShare(ctx, lsAdded, caches.AvailableEntities, caches.OldLinkShareIDToNewID) lsAdded = filterUnavailableEntitiesInLinkShare(ctx, lsAdded, caches.AvailableEntities, caches.OldLinkShareIDToNewID)
// Link shares have to be updated before permissions as we have to // Link shares have to be updated before permissions as we have to
@ -608,7 +609,7 @@ func RestorePermissions(
return return
} }
permAdded, permRemoved := metadata.DiffPermissions(previous.Permissions, current.Permissions) permAdded, permRemoved := odmetadata.DiffPermissions(previous.Permissions, current.Permissions)
permAdded = filterUnavailableEntitiesInPermissions(ctx, permAdded, caches.AvailableEntities, caches.OldPermIDToNewID) permAdded = filterUnavailableEntitiesInPermissions(ctx, permAdded, caches.AvailableEntities, caches.OldPermIDToNewID)
if didReset { if didReset {
@ -617,7 +618,7 @@ func RestorePermissions(
// that an item has as they too will be removed. // that an item has as they too will be removed.
logger.Ctx(ctx).Debug("link share creation reset all inherited permissions") logger.Ctx(ctx).Debug("link share creation reset all inherited permissions")
permRemoved = []metadata.Permission{} permRemoved = []odmetadata.Permission{}
permAdded = current.Permissions permAdded = current.Permissions
} }

View File

@ -17,7 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/operations/inject"
@ -31,6 +31,7 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/custom" "github.com/alcionai/corso/src/pkg/services/m365/custom"
) )
@ -552,7 +553,7 @@ func CreateRestoreFolders(
drivePath *path.DrivePath, drivePath *path.DrivePath,
restoreDir *path.Builder, restoreDir *path.Builder,
folderPath path.Path, folderPath path.Path,
folderMetadata metadata.Metadata, folderMetadata odmetadata.Metadata,
caches *restoreCaches, caches *restoreCaches,
restorePerms bool, restorePerms bool,
errs *fault.Bus, errs *fault.Bus,
@ -876,12 +877,12 @@ func FetchAndReadMetadata(
ctx context.Context, ctx context.Context,
fibn data.FetchItemByNamer, fibn data.FetchItemByNamer,
metaName string, metaName string,
) (metadata.Metadata, error) { ) (odmetadata.Metadata, error) {
ctx = clues.Add(ctx, "meta_file_name", metaName) ctx = clues.Add(ctx, "meta_file_name", metaName)
metaFile, err := fibn.FetchItemByName(ctx, metaName) metaFile, err := fibn.FetchItemByName(ctx, metaName)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.Wrap(err, "getting item metadata") return odmetadata.Metadata{}, clues.Wrap(err, "getting item metadata")
} }
metaReader := metaFile.ToReader() metaReader := metaFile.ToReader()
@ -889,25 +890,25 @@ func FetchAndReadMetadata(
meta, err := getMetadata(metaReader) meta, err := getMetadata(metaReader)
if err != nil { if err != nil {
return metadata.Metadata{}, clues.Wrap(err, "deserializing item metadata") return odmetadata.Metadata{}, clues.Wrap(err, "deserializing item metadata")
} }
return meta, nil return meta, nil
} }
// getMetadata read and parses the metadata info for an item // getMetadata read and parses the metadata info for an item
func getMetadata(metar io.ReadCloser) (metadata.Metadata, error) { func getMetadata(metar io.ReadCloser) (odmetadata.Metadata, error) {
var meta metadata.Metadata var meta odmetadata.Metadata
// `metar` will be nil for the top level container folder // `metar` will be nil for the top level container folder
if metar != nil { if metar != nil {
metaraw, err := io.ReadAll(metar) metaraw, err := io.ReadAll(metar)
if err != nil { if err != nil {
return metadata.Metadata{}, err return odmetadata.Metadata{}, err
} }
err = json.Unmarshal(metaraw, &meta) err = json.Unmarshal(metaraw, &meta)
if err != nil { if err != nil {
return metadata.Metadata{}, err return odmetadata.Metadata{}, err
} }
} }

View File

@ -18,7 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/readers" "github.com/alcionai/corso/src/internal/common/readers"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub" m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
@ -29,6 +29,7 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
func testElementsMatch[T any]( func testElementsMatch[T any](
@ -693,7 +694,7 @@ func compareExchangeEvent(
checkEvent(t, expectedEvent, itemEvent) checkEvent(t, expectedEvent, itemEvent)
} }
func permissionEqual(expected metadata.Permission, got metadata.Permission) bool { func permissionEqual(expected odmetadata.Permission, got odmetadata.Permission) bool {
if !strings.EqualFold(expected.Email, got.Email) { if !strings.EqualFold(expected.Email, got.Email) {
return false return false
} }
@ -722,7 +723,7 @@ func permissionEqual(expected metadata.Permission, got metadata.Permission) bool
return true return true
} }
func linkSharesEqual(expected metadata.LinkShare, got metadata.LinkShare) bool { func linkSharesEqual(expected odmetadata.LinkShare, got odmetadata.LinkShare) bool {
if !strings.EqualFold(expected.Link.Scope, got.Link.Scope) { if !strings.EqualFold(expected.Link.Scope, got.Link.Scope) {
return false return false
} }
@ -815,8 +816,8 @@ func compareDriveItem(
if isMeta { if isMeta {
var ( var (
itemMeta metadata.Metadata itemMeta odmetadata.Metadata
expectedMeta metadata.Metadata expectedMeta odmetadata.Metadata
) )
err = json.Unmarshal(buf, &itemMeta) err = json.Unmarshal(buf, &itemMeta)
@ -859,7 +860,7 @@ func compareDriveItem(
assert.Equal(t, expectedMeta.SharingMode, itemMeta.SharingMode, "sharing mode") assert.Equal(t, expectedMeta.SharingMode, itemMeta.SharingMode, "sharing mode")
// We cannot restore owner permissions, so skip checking them // We cannot restore owner permissions, so skip checking them
itemPerms := []metadata.Permission{} itemPerms := []odmetadata.Permission{}
for _, p := range itemMeta.Permissions { for _, p := range itemMeta.Permissions {
if p.Roles[0] != "owner" { if p.Roles[0] != "owner" {

View File

@ -8,41 +8,42 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub" m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
// For any version post this(inclusive), we expect to be using IDs for // For any version post this(inclusive), we expect to be using IDs for
// permission instead of email // permission instead of email
const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions const versionPermissionSwitchedToID = version.OneDrive4DirIncludesPermissions
func getMetadata(fileName string, meta MetaData, permUseID bool) metadata.Metadata { func getMetadata(fileName string, meta MetaData, permUseID bool) odmetadata.Metadata {
if meta.SharingMode != metadata.SharingModeCustom { if meta.SharingMode != odmetadata.SharingModeCustom {
return metadata.Metadata{ return odmetadata.Metadata{
FileName: fileName, FileName: fileName,
SharingMode: meta.SharingMode, SharingMode: meta.SharingMode,
} }
} }
testMeta := metadata.Metadata{FileName: fileName} testMeta := odmetadata.Metadata{FileName: fileName}
if len(meta.Perms.User) != 0 { if len(meta.Perms.User) != 0 {
// In case of permissions, the id will usually be same for same // In case of permissions, the id will usually be same for same
// user/role combo unless deleted and readded, but we have to do // user/role combo unless deleted and readded, but we have to do
// this as we only have two users of which one is already taken. // this as we only have two users of which one is already taken.
id := uuid.NewString() id := uuid.NewString()
uperm := metadata.Permission{ID: id, Roles: meta.Perms.Roles} uperm := odmetadata.Permission{ID: id, Roles: meta.Perms.Roles}
uperm.Email = meta.Perms.User uperm.Email = meta.Perms.User
if permUseID { if permUseID {
uperm.EntityID = meta.Perms.EntityID uperm.EntityID = meta.Perms.EntityID
} }
testMeta.Permissions = []metadata.Permission{uperm} testMeta.Permissions = []odmetadata.Permission{uperm}
} }
if len(meta.LinkShares) != 0 { if len(meta.LinkShares) != 0 {
@ -61,9 +62,9 @@ func getMetadata(fileName string, meta MetaData, permUseID bool) metadata.Metada
roles = []string{"read"} roles = []string{"read"}
} }
ls := metadata.LinkShare{ ls := odmetadata.LinkShare{
ID: id, // id is required for mapping from parent ID: id, // id is required for mapping from parent
Link: metadata.LinkShareLink{ Link: odmetadata.LinkShareLink{
Scope: ls.Scope, Scope: ls.Scope,
Type: ls.Type, Type: ls.Type,
WebURL: id, WebURL: id,
@ -86,13 +87,13 @@ type PermData struct {
} }
type LinkShareData struct { type LinkShareData struct {
Entities []metadata.Entity Entities []odmetadata.Entity
Scope string Scope string
Type string Type string
} }
type MetaData struct { type MetaData struct {
SharingMode metadata.SharingMode SharingMode odmetadata.SharingMode
Perms PermData Perms PermData
LinkShares []LinkShareData LinkShares []LinkShareData
} }

View File

@ -9,10 +9,10 @@ import (
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
type ColInfo struct { type ColInfo struct {

View File

@ -7,8 +7,8 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
// Max number of items for which we will print details. If there are // Max number of items for which we will print details. If there are

View File

@ -13,12 +13,12 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/dttm" "github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
) )
// ------------------------------------------------------------ // ------------------------------------------------------------

View File

@ -10,7 +10,6 @@ import (
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/streamstore"
@ -19,6 +18,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
"github.com/alcionai/corso/src/pkg/store" "github.com/alcionai/corso/src/pkg/store"
) )

View File

@ -1,17 +1,16 @@
package metadata package metadata
import ( import (
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
func IsMetadataFile(p path.Path) bool { func IsMetadataFile(p path.Path) bool {
switch p.Service() { switch p.Service() {
case path.OneDriveService: case path.OneDriveService:
return metadata.HasMetaSuffix(p.Item()) return HasMetaSuffix(p.Item())
case path.SharePointService, path.GroupsService: case path.SharePointService, path.GroupsService:
return p.Category() == path.LibrariesCategory && metadata.HasMetaSuffix(p.Item()) return p.Category() == path.LibrariesCategory && HasMetaSuffix(p.Item())
default: default:
return false return false

View File

@ -9,7 +9,6 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api/graph/metadata"
@ -29,12 +28,12 @@ var (
notMetaSuffixes = []string{ notMetaSuffixes = []string{
"", "",
odmetadata.DataFileSuffix, metadata.DataFileSuffix,
} }
metaSuffixes = []string{ metaSuffixes = []string{
odmetadata.MetaFileSuffix, metadata.MetaFileSuffix,
odmetadata.DirMetaFileSuffix, metadata.DirMetaFileSuffix,
} }
cases = []testCase{ cases = []testCase{