larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add locationRef to details

Browse Source 
Adds a new reference to the details ent: location-
ref.  The location holds the human-readable
version of the item's location in whatever m365
service sourced the item.  Hookup is incomplete,
following PRs will fill out functionality.

Also adds a LocationPather interface to data_
collections to pass this data back and forth
between producers and consumers.
This commit is contained in:
ryanfkeepers 2023-02-07 12:28:43 -07:00
parent 47f5ca1d95
commit 1b1629e09c
10 changed files with 223 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/internal/connector/exchange/container_resolver_test.go
View File

@ -595,7 +595,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := CreateContainerDestinaion(
folderID, err := CreateContainerDestination(
ctx,
m365,
test.pathFunc1(t),
@ -608,7 +608,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := CreateContainerDestinaion(
secondID, err := CreateContainerDestination(
ctx,
m365,
test.pathFunc2(t),

12
src/internal/connector/exchange/service_restore.go
View File

@ -344,7 +344,7 @@ func RestoreExchangeDataCollections(
userCaches = directoryCaches[userID]
}
containerID, err := CreateContainerDestinaion(
containerID, err := CreateContainerDestination(
ctx,
creds,
dc.FullPath(),
@ -447,10 +447,16 @@ func restoreCollection(
continue
}
// var locationRef string
// if category == path.ContactsCategory {
// locationRef = itemPath.Folder()
// }
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
"", // TODO: locationRef
true,
details.ItemInfo{
Exchange: info,
@ -461,12 +467,12 @@ func restoreCollection(
}
}
// CreateContainerDestinaion builds the destination into the container
// CreateContainerDestination builds the destination into the container
// at the provided path. As a precondition, the destination cannot
// already exist. If it does then an error is returned. The provided
// containerResolver is updated with the new destination.
// @ returns the container ID of the new destination container.
func CreateContainerDestinaion(
func CreateContainerDestination(
ctx context.Context,
creds account.M365Config,
directory path.Path,

16
src/internal/connector/onedrive/restore.go
View File

@ -235,7 +235,13 @@ func RestoreCollection(
restoredIDs[trimmedName] = itemID
deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo)
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
"", // TODO: implement locationRef
true,
itemInfo)
// Mark it as success without processing .meta
// file if we are not restoring permissions
@ -343,7 +349,13 @@ func RestoreCollection(
continue
}
deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo)
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
"", // TODO: implement locationRef
true,
itemInfo)
metrics.Successes++
}
}

2
src/internal/connector/sharepoint/restore.go
View File

@ -276,6 +276,7 @@ func RestoreListCollection(
itemPath.String(),
itemPath.ShortRef(),
"",
"", // TODO: implement locationRef
true,
itemInfo)
@ -355,6 +356,7 @@ func RestorePageCollection(
itemPath.String(),
itemPath.ShortRef(),
"",
"", // TODO: implement locationRef
true,
itemInfo,
)

6
src/internal/data/data_collection.go
View File

@ -92,6 +92,12 @@ type Stream interface {
Deleted() bool
}
// Locationer provides a LocationPath describing the path with Display Names
// instead of canonical IDs
type LocationPather interface {
LocationPath() path.Path
}
// StreamInfo is used to provide service specific
// information about the Stream
type StreamInfo interface {

35
src/internal/kopia/upload.go
View File

@ -124,10 +124,11 @@ func (rw *restoreStreamReader) Read(p []byte) (n int, err error) {
}
type itemDetails struct {
info *details.ItemInfo
repoPath path.Path
prevPath path.Path
cached bool
info *details.ItemInfo
repoPath path.Path
prevPath path.Path
locationPath path.Path
cached bool
}
type corsoProgress struct {
@ -188,20 +189,24 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
parent := d.repoPath.ToBuilder().Dir()
var locationFolders string
if d.locationPath != nil {
locationFolders = d.locationPath.Folder()
}
cp.deets.Add(
d.repoPath.String(),
d.repoPath.ShortRef(),
parent.ShortRef(),
locationFolders,
!d.cached,
*d.info,
)
*d.info)
folders := details.FolderEntriesForPath(parent)
folders := details.FolderEntriesForPath(parent, d.locationPath.ToBuilder())
cp.deets.AddFoldersForItem(
folders,
*d.info,
!d.cached,
)
!d.cached)
}
// Kopia interface function used as a callback when kopia finishes hashing a file.
@ -311,6 +316,12 @@ func collectionEntries(
continue
}
var locationPath path.Path
if lp, ok := e.(data.LocationPather); ok {
locationPath = lp.LocationPath()
}
trace.Log(ctx, "kopia:streamEntries:item", itemPath.String())
if e.Deleted() {
@ -332,7 +343,11 @@ func collectionEntries(
// previous snapshot then we should populate prevPath here and leave
// info nil.
itemInfo := ei.Info()
d := &itemDetails{info: &itemInfo, repoPath: itemPath}
d := &itemDetails{
info: &itemInfo,
repoPath: itemPath,
locationPath: locationPath,
}
progress.put(encodeAsPath(itemPath.PopFront().Elements()...), d)
}

3
src/internal/operations/backup.go
View File

@ -581,10 +581,11 @@ func mergeDetails(
newPath.String(),
newPath.ShortRef(),
newPath.ToBuilder().Dir().ShortRef(),
"", // TODO Location Ref,
itemUpdated,
item)
folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir())
folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir(), nil)
deets.AddFoldersForItem(folders, item, itemUpdated)
// Track how many entries we added so that we know if we got them all when

3
src/internal/streamstore/streamstore_test.go
View File

@ -44,7 +44,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() {
deetsBuilder := &details.Builder{}
deetsBuilder.Add("ref", "shortref", "parentref", true,
deetsBuilder.Add("ref", "shortref", "parentref", "locationRef", true,
details.ItemInfo{
Exchange: &details.ExchangeInfo{
Subject: "hello world",
@ -66,6 +66,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() {
assert.Equal(t, deets.Entries[0].ParentRef, readDeets.Entries[0].ParentRef)
assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef)
assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef)
assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef)
assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated)
assert.NotNil(t, readDeets.Entries[0].Exchange)
assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange)

71
src/pkg/backup/details/details.go
View File

@ -15,11 +15,12 @@ import (
)
type folderEntry struct {
RepoRef string
ShortRef string
ParentRef string
Updated bool
Info ItemInfo
RepoRef string
ShortRef string
ParentRef string
LocationRef string
Updated bool
Info ItemInfo
}
// --------------------------------------------------------------------------------
@ -110,10 +111,14 @@ type Builder struct {
knownFolders map[string]folderEntry `json:"-"`
}
func (b *Builder) Add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
func (b *Builder) Add(
repoRef, shortRef, parentRef, locationRef string,
updated bool,
info ItemInfo,
) {
b.mu.Lock()
defer b.mu.Unlock()
b.d.add(repoRef, shortRef, parentRef, updated, info)
b.d.add(repoRef, shortRef, parentRef, locationRef, updated, info)
}
func (b *Builder) Details() *Details {
@ -131,16 +136,23 @@ func (b *Builder) Details() *Details {
// TODO(ashmrtn): If we never need to pre-populate the modified time of a folder
// we should just merge this with AddFoldersForItem, have Add call
// AddFoldersForItem, and unexport AddFoldersForItem.
func FolderEntriesForPath(parent *path.Builder) []folderEntry {
func FolderEntriesForPath(parent, location *path.Builder) []folderEntry {
folders := []folderEntry{}
lfs := locationRefOf(location)
for len(parent.Elements()) > 0 {
nextParent := parent.Dir()
var lr string
if lfs != nil {
lr = lfs.String()
}
folders = append(folders, folderEntry{
RepoRef: parent.String(),
ShortRef: parent.ShortRef(),
ParentRef: nextParent.ShortRef(),
RepoRef: parent.String(),
ShortRef: parent.ShortRef(),
ParentRef: nextParent.ShortRef(),
LocationRef: lr,
Info: ItemInfo{
Folder: &FolderInfo{
ItemType: FolderItem,
@ -150,11 +162,30 @@ func FolderEntriesForPath(parent *path.Builder) []folderEntry {
})
parent = nextParent
if lfs != nil {
lfs = lfs.Dir()
}
}
return folders
}
// assumes the pb contains a path like:
// <tenant>/<service>/<owner>/<category>/<logical_containers>...
// and returns a string with only <logical_containers>/...
func locationRefOf(pb *path.Builder) *path.Builder {
if pb == nil {
return nil
}
for i := 0; i < 4; i++ {
pb = pb.PopFront()
}
return pb
}
// AddFoldersForItem adds entries for the given folders. It skips adding entries that
// have been added by previous calls.
func (b *Builder) AddFoldersForItem(folders []folderEntry, itemInfo ItemInfo, updated bool) {
@ -202,7 +233,11 @@ type Details struct {
DetailsModel
}
func (d *Details) add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
func (d *Details) add(
repoRef, shortRef, parentRef, locationRef string,
updated bool,
info ItemInfo,
) {
d.Entries = append(d.Entries, DetailsEntry{
RepoRef: repoRef,
ShortRef: shortRef,
@ -233,9 +268,21 @@ type DetailsEntry struct {
RepoRef string `json:"repoRef"`
ShortRef string `json:"shortRef"`
ParentRef string `json:"parentRef,omitempty"`
// LocationRef contains the logical path structure by its human-readable
// display names. IE: If an item is located at "/Inbox/Important", we
// hold that string in the LocationRef, while the actual IDs of each
// container are used for the RepoRef.
// LocationRef only holds the container values, and does not include
// the metadata prefixes (tenant, service, owner, etc) found in the
// repoRef.
// Currently only implemented for Exchange Calendars.
LocationRef string `json:"locationRef,omitempty"`
// Indicates the item was added or updated in this backup
// Always `true` for full backups
Updated bool `json:"updated"`
ItemInfo
}

165
src/pkg/backup/details/details_test.go
View File

@ -39,8 +39,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "no info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
},
expectHs: []string{"ID"},
expectVs: []string{"deadbeef"},
@ -48,8 +49,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "exchange event info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeEvent,
@ -67,8 +69,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "exchange contact info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeContact,
@ -82,8 +85,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "exchange mail info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemInfo: ItemInfo{
Exchange: &ExchangeInfo{
ItemType: ExchangeMail,
@ -99,8 +103,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "sharepoint info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemInfo: ItemInfo{
SharePoint: &SharePointInfo{
ItemName: "itemName",
@ -128,8 +133,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
{
name: "oneDrive info",
entry: DetailsEntry{
RepoRef: "reporef",
ShortRef: "deadbeef",
RepoRef: "reporef",
ShortRef: "deadbeef",
LocationRef: "locationref",
ItemInfo: ItemInfo{
OneDrive: &OneDriveInfo{
ItemName: "itemName",
@ -157,37 +163,57 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
}
var pathItemsTable = []struct {
name string
ents []DetailsEntry
expectRefs []string
name string
ents []DetailsEntry
expectRepoRefs []string
expectLocationRefs []string
}{
{
name: "nil entries",
ents: nil,
expectRefs: []string{},
name: "nil entries",
ents: nil,
expectRepoRefs: []string{},
expectLocationRefs: []string{},
},
{
name: "single entry",
ents: []DetailsEntry{
{RepoRef: "abcde"},
{
RepoRef: "abcde",
LocationRef: "locationref",
},
},
expectRefs: []string{"abcde"},
expectRepoRefs: []string{"abcde"},
expectLocationRefs: []string{"locationref"},
},
{
name: "multiple entries",
ents: []DetailsEntry{
{RepoRef: "abcde"},
{RepoRef: "12345"},
{
RepoRef: "abcde",
LocationRef: "locationref",
},
{
RepoRef: "12345",
LocationRef: "locationref2",
},
},
expectRefs: []string{"abcde", "12345"},
expectRepoRefs: []string{"abcde", "12345"},
expectLocationRefs: []string{"locationref", "locationref2"},
},
{
name: "multiple entries with folder",
ents: []DetailsEntry{
{RepoRef: "abcde"},
{RepoRef: "12345"},
{
RepoRef: "deadbeef",
RepoRef: "abcde",
LocationRef: "locationref",
},
{
RepoRef: "12345",
LocationRef: "locationref2",
},
{
RepoRef: "deadbeef",
LocationRef: "locationref3",
ItemInfo: ItemInfo{
Folder: &FolderInfo{
DisplayName: "test folder",
@ -195,7 +221,8 @@ var pathItemsTable = []struct {
},
},
},
expectRefs: []string{"abcde", "12345"},
expectRepoRefs: []string{"abcde", "12345"},
expectLocationRefs: []string{"locationref", "locationref2"},
},
}
@ -207,7 +234,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
Entries: test.ents,
},
}
assert.Equal(t, test.expectRefs, d.Paths())
assert.Equal(t, test.expectRepoRefs, d.Paths())
})
}
}
@ -222,10 +249,11 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Items() {
}
ents := d.Items()
assert.Len(t, ents, len(test.expectRefs))
assert.Len(t, ents, len(test.expectRepoRefs))
for _, e := range ents {
assert.Contains(t, test.expectRefs, e.RepoRef)
assert.Contains(t, test.expectRepoRefs, e.RepoRef)
assert.Contains(t, test.expectLocationRefs, e.LocationRef)
}
})
}
@ -253,9 +281,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
name: "MultipleFolders",
folders: []folderEntry{
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeOlderThanItem,
@ -263,9 +292,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
},
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
LocationRef: "lr2",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeNewerThanItem,
@ -283,9 +313,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
name: "MultipleFoldersWithRepeats",
folders: []folderEntry{
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeOlderThanItem,
@ -293,9 +324,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
},
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
LocationRef: "lr2",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeOlderThanItem,
@ -303,9 +335,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
},
},
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeOlderThanItem,
@ -313,9 +346,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
},
},
{
RepoRef: "rr3",
ShortRef: "sr3",
ParentRef: "pr3",
RepoRef: "rr3",
ShortRef: "sr3",
ParentRef: "pr3",
LocationRef: "lr3",
Info: ItemInfo{
Folder: &FolderInfo{
Modified: folderTimeNewerThanItem,
@ -363,18 +397,20 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() {
name: "ItemNotUpdated_NoChange",
folders: []folderEntry{
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{},
},
Updated: true,
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
LocationRef: "lr2",
Info: ItemInfo{
Folder: &FolderInfo{},
},
@ -390,17 +426,19 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() {
name: "ItemUpdated",
folders: []folderEntry{
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{},
},
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
LocationRef: "lr2",
Info: ItemInfo{
Folder: &FolderInfo{},
},
@ -482,9 +520,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersDifferentServices() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
folder := folderEntry{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
LocationRef: "lr1",
Info: ItemInfo{
Folder: &FolderInfo{},
},