add locationRef to details

Adds a new reference to the details ent: location-
ref.  The location holds the human-readable
version of the item's location in whatever m365
service sourced the item.  Hookup is incomplete,
following PRs will fill out functionality.

Also adds a LocationPather interface to data_
collections to pass this data back and forth
between producers and consumers.

src/internal/connector/exchange/container_resolver_test.go

@@ -595,7 +595,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
 
 	for _, test := range tests {
 		suite.T().Run(test.name, func(t *testing.T) {
-			folderID, err := CreateContainerDestinaion(
+			folderID, err := CreateContainerDestination(
 				ctx,
 				m365,
 				test.pathFunc1(t),
@@ -608,7 +608,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
 			_, err = resolver.IDToPath(ctx, folderID)
 			assert.NoError(t, err)
 
-			secondID, err := CreateContainerDestinaion(
+			secondID, err := CreateContainerDestination(
 				ctx,
 				m365,
 				test.pathFunc2(t),

src/internal/connector/exchange/service_restore.go

@@ -344,7 +344,7 @@ func RestoreExchangeDataCollections(
 			userCaches = directoryCaches[userID]
 		}
 
-		containerID, err := CreateContainerDestinaion(
+		containerID, err := CreateContainerDestination(
 			ctx,
 			creds,
 			dc.FullPath(),
@@ -447,10 +447,16 @@ func restoreCollection(
 				continue
 			}
 
+			// var locationRef string
+			// if category == path.ContactsCategory {
+			// 	locationRef = itemPath.Folder()
+			// }
+
 			deets.Add(
 				itemPath.String(),
 				itemPath.ShortRef(),
 				"",
+				"", // TODO: locationRef
 				true,
 				details.ItemInfo{
 					Exchange: info,
@@ -461,12 +467,12 @@ func restoreCollection(
 	}
 }
 
-// CreateContainerDestinaion builds the destination into the container
+// CreateContainerDestination builds the destination into the container
 // at the provided path.  As a precondition, the destination cannot
 // already exist.  If it does then an error is returned.  The provided
 // containerResolver is updated with the new destination.
 // @ returns the container ID of the new destination container.
-func CreateContainerDestinaion(
+func CreateContainerDestination(
 	ctx context.Context,
 	creds account.M365Config,
 	directory path.Path,

src/internal/connector/onedrive/restore.go

@@ -235,7 +235,13 @@ func RestoreCollection(
 
 					restoredIDs[trimmedName] = itemID
 
-					deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo)
+					deets.Add(
+						itemPath.String(),
+						itemPath.ShortRef(),
+						"",
+						"", // TODO: implement locationRef
+						true,
+						itemInfo)
 
 					// Mark it as success without processing .meta
 					// file if we are not restoring permissions
@@ -343,7 +349,13 @@ func RestoreCollection(
 					continue
 				}
 
-				deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo)
+				deets.Add(
+					itemPath.String(),
+					itemPath.ShortRef(),
+					"",
+					"", // TODO: implement locationRef
+					true,
+					itemInfo)
 				metrics.Successes++
 			}
 		}

src/internal/connector/sharepoint/restore.go

@@ -276,6 +276,7 @@ func RestoreListCollection(
 				itemPath.String(),
 				itemPath.ShortRef(),
 				"",
+				"", // TODO: implement locationRef
 				true,
 				itemInfo)
 
@@ -355,6 +356,7 @@ func RestorePageCollection(
 				itemPath.String(),
 				itemPath.ShortRef(),
 				"",
+				"", // TODO: implement locationRef
 				true,
 				itemInfo,
 			)

src/internal/data/data_collection.go

@@ -92,6 +92,12 @@ type Stream interface {
 	Deleted() bool
 }
 
+// Locationer provides a LocationPath describing the path with Display Names
+// instead of canonical IDs
+type LocationPather interface {
+	LocationPath() path.Path
+}
+
 // StreamInfo is used to provide service specific
 // information about the Stream
 type StreamInfo interface {

src/internal/kopia/upload.go

@@ -124,10 +124,11 @@ func (rw *restoreStreamReader) Read(p []byte) (n int, err error) {
 }
 
 type itemDetails struct {
-	info     *details.ItemInfo
+	info         *details.ItemInfo
-	repoPath path.Path
+	repoPath     path.Path
-	prevPath path.Path
+	prevPath     path.Path
-	cached   bool
+	locationPath path.Path
+	cached       bool
 }
 
 type corsoProgress struct {
@@ -188,20 +189,24 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
 
 	parent := d.repoPath.ToBuilder().Dir()
 
+	var locationFolders string
+	if d.locationPath != nil {
+		locationFolders = d.locationPath.Folder()
+	}
+
 	cp.deets.Add(
 		d.repoPath.String(),
 		d.repoPath.ShortRef(),
 		parent.ShortRef(),
+		locationFolders,
 		!d.cached,
-		*d.info,
+		*d.info)
-	)
 
-	folders := details.FolderEntriesForPath(parent)
+	folders := details.FolderEntriesForPath(parent, d.locationPath.ToBuilder())
 	cp.deets.AddFoldersForItem(
 		folders,
 		*d.info,
-		!d.cached,
+		!d.cached)
-	)
 }
 
 // Kopia interface function used as a callback when kopia finishes hashing a file.
@@ -311,6 +316,12 @@ func collectionEntries(
 				continue
 			}
 
+			var locationPath path.Path
+
+			if lp, ok := e.(data.LocationPather); ok {
+				locationPath = lp.LocationPath()
+			}
+
 			trace.Log(ctx, "kopia:streamEntries:item", itemPath.String())
 
 			if e.Deleted() {
@@ -332,7 +343,11 @@ func collectionEntries(
 				// previous snapshot then we should populate prevPath here and leave
 				// info nil.
 				itemInfo := ei.Info()
-				d := &itemDetails{info: &itemInfo, repoPath: itemPath}
+				d := &itemDetails{
+					info:         &itemInfo,
+					repoPath:     itemPath,
+					locationPath: locationPath,
+				}
 				progress.put(encodeAsPath(itemPath.PopFront().Elements()...), d)
 			}
 

src/internal/operations/backup.go

@@ -581,10 +581,11 @@ func mergeDetails(
 				newPath.String(),
 				newPath.ShortRef(),
 				newPath.ToBuilder().Dir().ShortRef(),
+				"", // TODO Location Ref,
 				itemUpdated,
 				item)
 
-			folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir())
+			folders := details.FolderEntriesForPath(newPath.ToBuilder().Dir(), nil)
 			deets.AddFoldersForItem(folders, item, itemUpdated)
 
 			// Track how many entries we added so that we know if we got them all when

src/internal/streamstore/streamstore_test.go

@@ -44,7 +44,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() {
 
 	deetsBuilder := &details.Builder{}
 
-	deetsBuilder.Add("ref", "shortref", "parentref", true,
+	deetsBuilder.Add("ref", "shortref", "parentref", "locationRef", true,
 		details.ItemInfo{
 			Exchange: &details.ExchangeInfo{
 				Subject: "hello world",
@@ -66,6 +66,7 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() {
 	assert.Equal(t, deets.Entries[0].ParentRef, readDeets.Entries[0].ParentRef)
 	assert.Equal(t, deets.Entries[0].ShortRef, readDeets.Entries[0].ShortRef)
 	assert.Equal(t, deets.Entries[0].RepoRef, readDeets.Entries[0].RepoRef)
+	assert.Equal(t, deets.Entries[0].LocationRef, readDeets.Entries[0].LocationRef)
 	assert.Equal(t, deets.Entries[0].Updated, readDeets.Entries[0].Updated)
 	assert.NotNil(t, readDeets.Entries[0].Exchange)
 	assert.Equal(t, *deets.Entries[0].Exchange, *readDeets.Entries[0].Exchange)

src/pkg/backup/details/details.go

@@ -15,11 +15,12 @@ import (
 )
 
 type folderEntry struct {
-	RepoRef   string
+	RepoRef     string
-	ShortRef  string
+	ShortRef    string
-	ParentRef string
+	ParentRef   string
-	Updated   bool
+	LocationRef string
-	Info      ItemInfo
+	Updated     bool
+	Info        ItemInfo
 }
 
 // --------------------------------------------------------------------------------
@@ -110,10 +111,14 @@ type Builder struct {
 	knownFolders map[string]folderEntry `json:"-"`
 }
 
-func (b *Builder) Add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
+func (b *Builder) Add(
+	repoRef, shortRef, parentRef, locationRef string,
+	updated bool,
+	info ItemInfo,
+) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	b.d.add(repoRef, shortRef, parentRef, updated, info)
+	b.d.add(repoRef, shortRef, parentRef, locationRef, updated, info)
 }
 
 func (b *Builder) Details() *Details {
@@ -131,16 +136,23 @@ func (b *Builder) Details() *Details {
 // TODO(ashmrtn): If we never need to pre-populate the modified time of a folder
 // we should just merge this with AddFoldersForItem, have Add call
 // AddFoldersForItem, and unexport AddFoldersForItem.
-func FolderEntriesForPath(parent *path.Builder) []folderEntry {
+func FolderEntriesForPath(parent, location *path.Builder) []folderEntry {
 	folders := []folderEntry{}
+	lfs := locationRefOf(location)
 
 	for len(parent.Elements()) > 0 {
 		nextParent := parent.Dir()
 
+		var lr string
+		if lfs != nil {
+			lr = lfs.String()
+		}
+
 		folders = append(folders, folderEntry{
-			RepoRef:   parent.String(),
+			RepoRef:     parent.String(),
-			ShortRef:  parent.ShortRef(),
+			ShortRef:    parent.ShortRef(),
-			ParentRef: nextParent.ShortRef(),
+			ParentRef:   nextParent.ShortRef(),
+			LocationRef: lr,
 			Info: ItemInfo{
 				Folder: &FolderInfo{
 					ItemType:    FolderItem,
@@ -150,11 +162,30 @@ func FolderEntriesForPath(parent *path.Builder) []folderEntry {
 		})
 
 		parent = nextParent
+
+		if lfs != nil {
+			lfs = lfs.Dir()
+		}
 	}
 
 	return folders
 }
 
+// assumes the pb contains a path like:
+// <tenant>/<service>/<owner>/<category>/<logical_containers>...
+// and returns a string with only <logical_containers>/...
+func locationRefOf(pb *path.Builder) *path.Builder {
+	if pb == nil {
+		return nil
+	}
+
+	for i := 0; i < 4; i++ {
+		pb = pb.PopFront()
+	}
+
+	return pb
+}
+
 // AddFoldersForItem adds entries for the given folders. It skips adding entries that
 // have been added by previous calls.
 func (b *Builder) AddFoldersForItem(folders []folderEntry, itemInfo ItemInfo, updated bool) {
@@ -202,7 +233,11 @@ type Details struct {
 	DetailsModel
 }
 
-func (d *Details) add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
+func (d *Details) add(
+	repoRef, shortRef, parentRef, locationRef string,
+	updated bool,
+	info ItemInfo,
+) {
 	d.Entries = append(d.Entries, DetailsEntry{
 		RepoRef:   repoRef,
 		ShortRef:  shortRef,
@@ -233,9 +268,21 @@ type DetailsEntry struct {
 	RepoRef   string `json:"repoRef"`
 	ShortRef  string `json:"shortRef"`
 	ParentRef string `json:"parentRef,omitempty"`
+
+	// LocationRef contains the logical path structure by its human-readable
+	// display names.  IE:  If an item is located at "/Inbox/Important", we
+	// hold that string in the LocationRef, while the actual IDs of each
+	// container are used for the RepoRef.
+	// LocationRef only holds the container values, and does not include
+	// the metadata prefixes (tenant, service, owner, etc) found in the
+	// repoRef.
+	// Currently only implemented for Exchange Calendars.
+	LocationRef string `json:"locationRef,omitempty"`
+
 	// Indicates the item was added or updated in this backup
 	// Always `true` for full backups
 	Updated bool `json:"updated"`
+
 	ItemInfo
 }
 

src/pkg/backup/details/details_test.go

@@ -39,8 +39,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "no info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 			},
 			expectHs: []string{"ID"},
 			expectVs: []string{"deadbeef"},
@@ -48,8 +49,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "exchange event info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 				ItemInfo: ItemInfo{
 					Exchange: &ExchangeInfo{
 						ItemType:    ExchangeEvent,
@@ -67,8 +69,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "exchange contact info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 				ItemInfo: ItemInfo{
 					Exchange: &ExchangeInfo{
 						ItemType:    ExchangeContact,
@@ -82,8 +85,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "exchange mail info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 				ItemInfo: ItemInfo{
 					Exchange: &ExchangeInfo{
 						ItemType: ExchangeMail,
@@ -99,8 +103,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "sharepoint info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 				ItemInfo: ItemInfo{
 					SharePoint: &SharePointInfo{
 						ItemName:   "itemName",
@@ -128,8 +133,9 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 		{
 			name: "oneDrive info",
 			entry: DetailsEntry{
-				RepoRef:  "reporef",
+				RepoRef:     "reporef",
-				ShortRef: "deadbeef",
+				ShortRef:    "deadbeef",
+				LocationRef: "locationref",
 				ItemInfo: ItemInfo{
 					OneDrive: &OneDriveInfo{
 						ItemName:   "itemName",
@@ -157,37 +163,57 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 }
 
 var pathItemsTable = []struct {
-	name       string
+	name               string
-	ents       []DetailsEntry
+	ents               []DetailsEntry
-	expectRefs []string
+	expectRepoRefs     []string
+	expectLocationRefs []string
 }{
 	{
-		name:       "nil entries",
+		name:               "nil entries",
-		ents:       nil,
+		ents:               nil,
-		expectRefs: []string{},
+		expectRepoRefs:     []string{},
+		expectLocationRefs: []string{},
 	},
 	{
 		name: "single entry",
 		ents: []DetailsEntry{
-			{RepoRef: "abcde"},
+			{
+				RepoRef:     "abcde",
+				LocationRef: "locationref",
+			},
 		},
-		expectRefs: []string{"abcde"},
+		expectRepoRefs:     []string{"abcde"},
+		expectLocationRefs: []string{"locationref"},
 	},
 	{
 		name: "multiple entries",
 		ents: []DetailsEntry{
-			{RepoRef: "abcde"},
+			{
-			{RepoRef: "12345"},
+				RepoRef:     "abcde",
+				LocationRef: "locationref",
+			},
+			{
+				RepoRef:     "12345",
+				LocationRef: "locationref2",
+			},
 		},
-		expectRefs: []string{"abcde", "12345"},
+		expectRepoRefs:     []string{"abcde", "12345"},
+		expectLocationRefs: []string{"locationref", "locationref2"},
 	},
 	{
 		name: "multiple entries with folder",
 		ents: []DetailsEntry{
-			{RepoRef: "abcde"},
-			{RepoRef: "12345"},
 			{
-				RepoRef: "deadbeef",
+				RepoRef:     "abcde",
+				LocationRef: "locationref",
+			},
+			{
+				RepoRef:     "12345",
+				LocationRef: "locationref2",
+			},
+			{
+				RepoRef:     "deadbeef",
+				LocationRef: "locationref3",
 				ItemInfo: ItemInfo{
 					Folder: &FolderInfo{
 						DisplayName: "test folder",
@@ -195,7 +221,8 @@ var pathItemsTable = []struct {
 				},
 			},
 		},
-		expectRefs: []string{"abcde", "12345"},
+		expectRepoRefs:     []string{"abcde", "12345"},
+		expectLocationRefs: []string{"locationref", "locationref2"},
 	},
 }
 
@@ -207,7 +234,7 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
 					Entries: test.ents,
 				},
 			}
-			assert.Equal(t, test.expectRefs, d.Paths())
+			assert.Equal(t, test.expectRepoRefs, d.Paths())
 		})
 	}
 }
@@ -222,10 +249,11 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Items() {
 			}
 
 			ents := d.Items()
-			assert.Len(t, ents, len(test.expectRefs))
+			assert.Len(t, ents, len(test.expectRepoRefs))
 
 			for _, e := range ents {
-				assert.Contains(t, test.expectRefs, e.RepoRef)
+				assert.Contains(t, test.expectRepoRefs, e.RepoRef)
+				assert.Contains(t, test.expectLocationRefs, e.LocationRef)
 			}
 		})
 	}
@@ -253,9 +281,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 			name: "MultipleFolders",
 			folders: []folderEntry{
 				{
-					RepoRef:   "rr1",
+					RepoRef:     "rr1",
-					ShortRef:  "sr1",
+					ShortRef:    "sr1",
-					ParentRef: "pr1",
+					ParentRef:   "pr1",
+					LocationRef: "lr1",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeOlderThanItem,
@@ -263,9 +292,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 					},
 				},
 				{
-					RepoRef:   "rr2",
+					RepoRef:     "rr2",
-					ShortRef:  "sr2",
+					ShortRef:    "sr2",
-					ParentRef: "pr2",
+					ParentRef:   "pr2",
+					LocationRef: "lr2",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeNewerThanItem,
@@ -283,9 +313,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 			name: "MultipleFoldersWithRepeats",
 			folders: []folderEntry{
 				{
-					RepoRef:   "rr1",
+					RepoRef:     "rr1",
-					ShortRef:  "sr1",
+					ShortRef:    "sr1",
-					ParentRef: "pr1",
+					ParentRef:   "pr1",
+					LocationRef: "lr1",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeOlderThanItem,
@@ -293,9 +324,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 					},
 				},
 				{
-					RepoRef:   "rr2",
+					RepoRef:     "rr2",
-					ShortRef:  "sr2",
+					ShortRef:    "sr2",
-					ParentRef: "pr2",
+					ParentRef:   "pr2",
+					LocationRef: "lr2",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeOlderThanItem,
@@ -303,9 +335,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 					},
 				},
 				{
-					RepoRef:   "rr1",
+					RepoRef:     "rr1",
-					ShortRef:  "sr1",
+					ShortRef:    "sr1",
-					ParentRef: "pr1",
+					ParentRef:   "pr1",
+					LocationRef: "lr1",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeOlderThanItem,
@@ -313,9 +346,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 					},
 				},
 				{
-					RepoRef:   "rr3",
+					RepoRef:     "rr3",
-					ShortRef:  "sr3",
+					ShortRef:    "sr3",
-					ParentRef: "pr3",
+					ParentRef:   "pr3",
+					LocationRef: "lr3",
 					Info: ItemInfo{
 						Folder: &FolderInfo{
 							Modified: folderTimeNewerThanItem,
@@ -363,18 +397,20 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() {
 			name: "ItemNotUpdated_NoChange",
 			folders: []folderEntry{
 				{
-					RepoRef:   "rr1",
+					RepoRef:     "rr1",
-					ShortRef:  "sr1",
+					ShortRef:    "sr1",
-					ParentRef: "pr1",
+					ParentRef:   "pr1",
+					LocationRef: "lr1",
 					Info: ItemInfo{
 						Folder: &FolderInfo{},
 					},
 					Updated: true,
 				},
 				{
-					RepoRef:   "rr2",
+					RepoRef:     "rr2",
-					ShortRef:  "sr2",
+					ShortRef:    "sr2",
-					ParentRef: "pr2",
+					ParentRef:   "pr2",
+					LocationRef: "lr2",
 					Info: ItemInfo{
 						Folder: &FolderInfo{},
 					},
@@ -390,17 +426,19 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersUpdate() {
 			name: "ItemUpdated",
 			folders: []folderEntry{
 				{
-					RepoRef:   "rr1",
+					RepoRef:     "rr1",
-					ShortRef:  "sr1",
+					ShortRef:    "sr1",
-					ParentRef: "pr1",
+					ParentRef:   "pr1",
+					LocationRef: "lr1",
 					Info: ItemInfo{
 						Folder: &FolderInfo{},
 					},
 				},
 				{
-					RepoRef:   "rr2",
+					RepoRef:     "rr2",
-					ShortRef:  "sr2",
+					ShortRef:    "sr2",
-					ParentRef: "pr2",
+					ParentRef:   "pr2",
+					LocationRef: "lr2",
 					Info: ItemInfo{
 						Folder: &FolderInfo{},
 					},
@@ -482,9 +520,10 @@ func (suite *DetailsUnitSuite) TestDetails_AddFoldersDifferentServices() {
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
 			folder := folderEntry{
-				RepoRef:   "rr1",
+				RepoRef:     "rr1",
-				ShortRef:  "sr1",
+				ShortRef:    "sr1",
-				ParentRef: "pr1",
+				ParentRef:   "pr1",
+				LocationRef: "lr1",
 				Info: ItemInfo{
 					Folder: &FolderInfo{},
 				},