diff --git a/src/internal/connector/exchange/container_resolver_test.go b/src/internal/connector/exchange/container_resolver_test.go index be0704f46..c4c5bf50b 100644 --- a/src/internal/connector/exchange/container_resolver_test.go +++ b/src/internal/connector/exchange/container_resolver_test.go @@ -595,7 +595,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - folderID, err := CreateContainerDestinaion( + folderID, err := CreateContainerDestination( ctx, m365, test.pathFunc1(t), @@ -608,7 +608,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { _, err = resolver.IDToPath(ctx, folderID) assert.NoError(t, err) - secondID, err := CreateContainerDestinaion( + secondID, err := CreateContainerDestination( ctx, m365, test.pathFunc2(t), diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index bb0179c76..9f394accd 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -344,7 +344,7 @@ func RestoreExchangeDataCollections( userCaches = directoryCaches[userID] } - containerID, err := CreateContainerDestinaion( + containerID, err := CreateContainerDestination( ctx, creds, dc.FullPath(), @@ -447,10 +447,16 @@ func restoreCollection( continue } + // var locationRef string + // if category == path.ContactsCategory { + // locationRef = itemPath.Folder() + // } + deets.Add( itemPath.String(), itemPath.ShortRef(), "", + "", // TODO: locationRef true, details.ItemInfo{ Exchange: info, @@ -461,12 +467,12 @@ func restoreCollection( } } -// CreateContainerDestinaion builds the destination into the container +// CreateContainerDestination builds the destination into the container // at the provided path. As a precondition, the destination cannot // already exist. If it does then an error is returned. The provided // containerResolver is updated with the new destination. // @ returns the container ID of the new destination container. -func CreateContainerDestinaion( +func CreateContainerDestination( ctx context.Context, creds account.M365Config, directory path.Path, diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index e2029f4cc..a899b621d 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -235,7 +235,13 @@ func RestoreCollection( restoredIDs[trimmedName] = itemID - deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo) + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + "", // TODO: implement locationRef + true, + itemInfo) // Mark it as success without processing .meta // file if we are not restoring permissions @@ -343,7 +349,13 @@ func RestoreCollection( continue } - deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo) + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + "", // TODO: implement locationRef + true, + itemInfo) metrics.Successes++ } } diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index c2c92249f..a0996eb0e 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -276,6 +276,7 @@ func RestoreListCollection( itemPath.String(), itemPath.ShortRef(), "", + "", // TODO: implement locationRef true, itemInfo) @@ -355,6 +356,7 @@ func RestorePageCollection( itemPath.String(), itemPath.ShortRef(), "", + "", // TODO: implement locationRef true, itemInfo, ) diff --git a/src/internal/data/data_collection.go b/src/internal/data/data_collection.go index 794b4bc16..e6f3c5f43 100644 --- a/src/internal/data/data_collection.go +++ b/src/internal/data/data_collection.go @@ -92,6 +92,12 @@ type Stream interface { Deleted() bool } +// Locationer provides a LocationPath describing the path with Display Names +// instead of canonical IDs +type LocationPather interface { + LocationPath() path.Path +} + // StreamInfo is used to provide service specific // information about the Stream type StreamInfo interface { diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index a4ae1fbcc..7851df386 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -124,10 +124,11 @@ func (rw *restoreStreamReader) Read(p []byte) (n int, err error) { } type itemDetails struct { - info *details.ItemInfo - repoPath path.Path - prevPath path.Path - cached bool + info *details.ItemInfo + repoPath path.Path + prevPath path.Path + locationPath path.Path + cached bool } type corsoProgress struct { @@ -188,20 +189,24 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) { parent := d.repoPath.ToBuilder().Dir() + var locationFolders string + if d.locationPath != nil { + locationFolders = d.locationPath.Folder() + } + cp.deets.Add( d.repoPath.String(), d.repoPath.ShortRef(), parent.ShortRef(), + locationFolders, !d.cached, - *d.info, - ) + *d.info) - folders := details.FolderEntriesForPath(parent) + folders := details.FolderEntriesForPath(parent, d.locationPath.ToBuilder()) cp.deets.AddFoldersForItem( folders, *d.info, - !d.cached, - ) + !d.cached) } // Kopia interface function used as a callback when kopia finishes hashing a file. @@ -311,6 +316,12 @@ func collectionEntries( continue } + var locationPath path.Path + + if lp, ok := e.(data.LocationPather); ok { + locationPath = lp.LocationPath() + } + trace.Log(ctx, "kopia:streamEntries:item", itemPath.String()) if e.Deleted() { @@ -332,7 +343,11 @@ func collectionEntries( // previous snapshot then we should populate prevPath here and leave // info nil. itemInfo := ei.Info() - d := &itemDetails{info: &itemInfo, repoPath: itemPath} + d := &itemDetails{ + info: &itemInfo, + repoPath: itemPath, + locationPath: locationPath, + } progress.put(encodeAsPath(itemPath.PopFront().Elements()...), d) } diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index a82c0d30c..57af3b63f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -581,10 +581,11 @@ func mergeDetails( newPath.String(), newPath.ShortRef(), newPath.ToBuilder().Dir().ShortRef(), + "", // TODO Location Ref, itemUpdated, item) - folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir()) + folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir(), nil) deets.AddFoldersForItem(folders, item, itemUpdated) // Track how many entries we added so that we know if we got them all when diff --git a/src/internal/streamstore/streamstore_test.go b/src/internal/streamstore/streamstore_test.go index cc3309a11..c49dad147 100644 --- a/src/internal/streamstore/streamstore_test.go +++ b/src/internal/streamstore/streamstore_test.go @@ -44,7 +44,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() { deetsBuilder := &details.Builder{} - deetsBuilder.Add("ref", "shortref", "parentref", true, + deetsBuilder.Add("ref", "shortref", "parentref", "locationRef", true, details.ItemInfo{ Exchange: &details.ExchangeInfo{ Subject: "hello world", @@ -66,6 +66,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() { assert.Equal(t, deets.Entries[0].ParentRef, readDeets.Entries[0].ParentRef) assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef) assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef) + assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef) assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated) assert.NotNil(t, readDeets.Entries[0].Exchange) assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange) diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index bb392c223..4d0298437 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -15,11 +15,12 @@ import ( ) type folderEntry struct { - RepoRef string - ShortRef string - ParentRef string - Updated bool - Info ItemInfo + RepoRef string + ShortRef string + ParentRef string + LocationRef string + Updated bool + Info ItemInfo } // -------------------------------------------------------------------------------- @@ -110,10 +111,14 @@ type Builder struct { knownFolders map[string]folderEntry `json:"-"` } -func (b *Builder) Add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) { +func (b *Builder) Add( + repoRef, shortRef, parentRef, locationRef string, + updated bool, + info ItemInfo, +) { b.mu.Lock() defer b.mu.Unlock() - b.d.add(repoRef, shortRef, parentRef, updated, info) + b.d.add(repoRef, shortRef, parentRef, locationRef, updated, info) } func (b *Builder) Details() *Details { @@ -131,16 +136,23 @@ func (b *Builder) Details() *Details { // TODO(ashmrtn): If we never need to pre-populate the modified time of a folder // we should just merge this with AddFoldersForItem, have Add call // AddFoldersForItem, and unexport AddFoldersForItem. -func FolderEntriesForPath(parent *path.Builder) []folderEntry { +func FolderEntriesForPath(parent, location *path.Builder) []folderEntry { folders := []folderEntry{} + lfs := locationRefOf(location) for len(parent.Elements()) > 0 { nextParent := parent.Dir() + var lr string + if lfs != nil { + lr = lfs.String() + } + folders = append(folders, folderEntry{ - RepoRef: parent.String(), - ShortRef: parent.ShortRef(), - ParentRef: nextParent.ShortRef(), + RepoRef: parent.String(), + ShortRef: parent.ShortRef(), + ParentRef: nextParent.ShortRef(), + LocationRef: lr, Info: ItemInfo{ Folder: &FolderInfo{ ItemType: FolderItem, @@ -150,11 +162,30 @@ func FolderEntriesForPath(parent *path.Builder) []folderEntry { }) parent = nextParent + + if lfs != nil { + lfs = lfs.Dir() + } } return folders } +// assumes the pb contains a path like: +// ////... +// and returns a string with only /... +func locationRefOf(pb *path.Builder) *path.Builder { + if pb == nil { + return nil + } + + for i := 0; i < 4; i++ { + pb = pb.PopFront() + } + + return pb +} + // AddFoldersForItem adds entries for the given folders. It skips adding entries that // have been added by previous calls. func (b *Builder) AddFoldersForItem(folders []folderEntry, itemInfo ItemInfo, updated bool) { @@ -202,7 +233,11 @@ type Details struct { DetailsModel } -func (d *Details) add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) { +func (d *Details) add( + repoRef, shortRef, parentRef, locationRef string, + updated bool, + info ItemInfo, +) { d.Entries = append(d.Entries, DetailsEntry{ RepoRef: repoRef, ShortRef: shortRef, @@ -233,9 +268,21 @@ type DetailsEntry struct { RepoRef string `json:"repoRef"` ShortRef string `json:"shortRef"` ParentRef string `json:"parentRef,omitempty"` + + // LocationRef contains the logical path structure by its human-readable + // display names. IE: If an item is located at "/Inbox/Important", we + // hold that string in the LocationRef, while the actual IDs of each + // container are used for the RepoRef. + // LocationRef only holds the container values, and does not include + // the metadata prefixes (tenant, service, owner, etc) found in the + // repoRef. + // Currently only implemented for Exchange Calendars. + LocationRef string `json:"locationRef,omitempty"` + // Indicates the item was added or updated in this backup // Always `true` for full backups Updated bool `json:"updated"` + ItemInfo } diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index efc654246..41dcf1049 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -39,8 +39,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "no info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", }, expectHs: []string{"ID"}, expectVs: []string{"deadbeef"}, @@ -48,8 +49,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "exchange event info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeEvent, @@ -67,8 +69,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "exchange contact info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeContact, @@ -82,8 +85,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "exchange mail info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", ItemInfo: ItemInfo{ Exchange: &ExchangeInfo{ ItemType: ExchangeMail, @@ -99,8 +103,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "sharepoint info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", ItemInfo: ItemInfo{ SharePoint: &SharePointInfo{ ItemName: "itemName", @@ -128,8 +133,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { { name: "oneDrive info", entry: DetailsEntry{ - RepoRef: "reporef", - ShortRef: "deadbeef", + RepoRef: "reporef", + ShortRef: "deadbeef", + LocationRef: "locationref", ItemInfo: ItemInfo{ OneDrive: &OneDriveInfo{ ItemName: "itemName", @@ -157,37 +163,57 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { } var pathItemsTable = []struct { - name string - ents []DetailsEntry - expectRefs []string + name string + ents []DetailsEntry + expectRepoRefs []string + expectLocationRefs []string }{ { - name: "nil entries", - ents: nil, - expectRefs: []string{}, + name: "nil entries", + ents: nil, + expectRepoRefs: []string{}, + expectLocationRefs: []string{}, }, { name: "single entry", ents: []DetailsEntry{ - {RepoRef: "abcde"}, + { + RepoRef: "abcde", + LocationRef: "locationref", + }, }, - expectRefs: []string{"abcde"}, + expectRepoRefs: []string{"abcde"}, + expectLocationRefs: []string{"locationref"}, }, { name: "multiple entries", ents: []DetailsEntry{ - {RepoRef: "abcde"}, - {RepoRef: "12345"}, + { + RepoRef: "abcde", + LocationRef: "locationref", + }, + { + RepoRef: "12345", + LocationRef: "locationref2", + }, }, - expectRefs: []string{"abcde", "12345"}, + expectRepoRefs: []string{"abcde", "12345"}, + expectLocationRefs: []string{"locationref", "locationref2"}, }, { name: "multiple entries with folder", ents: []DetailsEntry{ - {RepoRef: "abcde"}, - {RepoRef: "12345"}, { - RepoRef: "deadbeef", + RepoRef: "abcde", + LocationRef: "locationref", + }, + { + RepoRef: "12345", + LocationRef: "locationref2", + }, + { + RepoRef: "deadbeef", + LocationRef: "locationref3", ItemInfo: ItemInfo{ Folder: &FolderInfo{ DisplayName: "test folder", @@ -195,7 +221,8 @@ var pathItemsTable = []struct { }, }, }, - expectRefs: []string{"abcde", "12345"}, + expectRepoRefs: []string{"abcde", "12345"}, + expectLocationRefs: []string{"locationref", "locationref2"}, }, } @@ -207,7 +234,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Path() { Entries: test.ents, }, } - assert.Equal(t, test.expectRefs, d.Paths()) + assert.Equal(t, test.expectRepoRefs, d.Paths()) }) } } @@ -222,10 +249,11 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Items() { } ents := d.Items() - assert.Len(t, ents, len(test.expectRefs)) + assert.Len(t, ents, len(test.expectRepoRefs)) for _, e := range ents { - assert.Contains(t, test.expectRefs, e.RepoRef) + assert.Contains(t, test.expectRepoRefs, e.RepoRef) + assert.Contains(t, test.expectLocationRefs, e.LocationRef) } }) } @@ -253,9 +281,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { name: "MultipleFolders", folders: []folderEntry{ { - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeOlderThanItem, @@ -263,9 +292,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { }, }, { - RepoRef: "rr2", - ShortRef: "sr2", - ParentRef: "pr2", + RepoRef: "rr2", + ShortRef: "sr2", + ParentRef: "pr2", + LocationRef: "lr2", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeNewerThanItem, @@ -283,9 +313,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { name: "MultipleFoldersWithRepeats", folders: []folderEntry{ { - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeOlderThanItem, @@ -293,9 +324,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { }, }, { - RepoRef: "rr2", - ShortRef: "sr2", - ParentRef: "pr2", + RepoRef: "rr2", + ShortRef: "sr2", + ParentRef: "pr2", + LocationRef: "lr2", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeOlderThanItem, @@ -303,9 +335,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { }, }, { - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeOlderThanItem, @@ -313,9 +346,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() { }, }, { - RepoRef: "rr3", - ShortRef: "sr3", - ParentRef: "pr3", + RepoRef: "rr3", + ShortRef: "sr3", + ParentRef: "pr3", + LocationRef: "lr3", Info: ItemInfo{ Folder: &FolderInfo{ Modified: folderTimeNewerThanItem, @@ -363,18 +397,20 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() { name: "ItemNotUpdated_NoChange", folders: []folderEntry{ { - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{}, }, Updated: true, }, { - RepoRef: "rr2", - ShortRef: "sr2", - ParentRef: "pr2", + RepoRef: "rr2", + ShortRef: "sr2", + ParentRef: "pr2", + LocationRef: "lr2", Info: ItemInfo{ Folder: &FolderInfo{}, }, @@ -390,17 +426,19 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() { name: "ItemUpdated", folders: []folderEntry{ { - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{}, }, }, { - RepoRef: "rr2", - ShortRef: "sr2", - ParentRef: "pr2", + RepoRef: "rr2", + ShortRef: "sr2", + ParentRef: "pr2", + LocationRef: "lr2", Info: ItemInfo{ Folder: &FolderInfo{}, }, @@ -482,9 +520,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersDifferentServices() { for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { folder := folderEntry{ - RepoRef: "rr1", - ShortRef: "sr1", - ParentRef: "pr1", + RepoRef: "rr1", + ShortRef: "sr1", + ParentRef: "pr1", + LocationRef: "lr1", Info: ItemInfo{ Folder: &FolderInfo{}, },