Create and populate folder backup details entries during backup (#872)

## Description

Other components may need to rebuild the directory hierarchy of items. As the paths Corso deals with can be hard to properly parse at times, store that information in the Corso backup details. The hierarchy can be rebuilt by following the `ParentRef` fields of items. The item at the root of the hierarchy has an empty `ParentRef` field.

Also hide these folders from end-users. They are not displayed during backup list nor are they eligible as a target for restore

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes #862
* closes #861
* closes #818 

merge after:
* #869 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/cli/backup/exchange_integration_test.go

@@ -270,11 +270,25 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeDetailsCmd() {
 			// compare the output
 			result := recorder.String()
 
-			for i, ent := range deets.Entries {
+			i := 0
+			foundFolders := 0
+
+			for _, ent := range deets.Entries {
+				// Skip folders as they don't mean anything to the end user.
+				if ent.Folder != nil {
+					foundFolders++
+					continue
+				}
+
 				t.Run(fmt.Sprintf("detail %d", i), func(t *testing.T) {
 					assert.Contains(t, result, ent.ShortRef)
 				})
+
+				i++
 			}
+
+			// At least the prefix of the path should be encoded as folders.
+			assert.Greater(suite.T(), foundFolders, 4)
 		})
 	}
 }

src/internal/kopia/wrapper.go

@@ -89,7 +89,35 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
 		return
 	}
 
-	cp.deets.Add(d.repoPath.String(), d.repoPath.ShortRef(), d.info)
+	parent := d.repoPath.ToBuilder().Dir()
+
+	cp.deets.Add(
+		d.repoPath.String(),
+		d.repoPath.ShortRef(),
+		parent.ShortRef(),
+		d.info,
+	)
+
+	folders := []details.FolderEntry{}
+
+	for len(parent.Elements()) > 0 {
+		nextParent := parent.Dir()
+
+		folders = append(folders, details.FolderEntry{
+			RepoRef:   parent.String(),
+			ShortRef:  parent.ShortRef(),
+			ParentRef: nextParent.ShortRef(),
+			Info: details.ItemInfo{
+				Folder: &details.FolderInfo{
+					DisplayName: parent.Elements()[len(parent.Elements())-1],
+				},
+			},
+		})
+
+		parent = nextParent
+	}
+
+	cp.deets.AddFolders(folders)
 }
 
 func (cp *corsoProgress) put(k string, v *itemDetails) {

src/internal/kopia/wrapper_test.go

@@ -132,20 +132,17 @@ func getDirEntriesForEntry(
 // ---------------
 type CorsoProgressUnitSuite struct {
 	suite.Suite
+	targetFilePath path.Path
+	targetFileName string
 }
 
 func TestCorsoProgressUnitSuite(t *testing.T) {
 	suite.Run(t, new(CorsoProgressUnitSuite))
 }
 
-func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
+func (suite *CorsoProgressUnitSuite) SetupSuite() {
-	type testInfo struct {
+	p, err := path.Builder{}.Append(
-		info *itemDetails
+		testInboxDir,
-		err  error
-	}
-
-	targetFilePath, err := path.Builder{}.Append(
-		"Inbox",
 		"testFile",
 	).ToDataLayerExchangePathForCategory(
 		testTenant,
@@ -155,11 +152,17 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 	)
 	require.NoError(suite.T(), err)
 
-	relativePath, err := targetFilePath.Dir()
+	suite.targetFilePath = p
-	require.NoError(suite.T(), err)
+	suite.targetFileName = suite.targetFilePath.ToBuilder().Dir().String()
+}
 
-	targetFileName := relativePath.String()
+func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
-	deets := &itemDetails{details.ItemInfo{}, targetFilePath}
+	type testInfo struct {
+		info *itemDetails
+		err  error
+	}
+
+	deets := &itemDetails{details.ItemInfo{}, suite.targetFilePath}
 
 	table := []struct {
 		name        string
@@ -170,17 +173,18 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 		{
 			name: "DetailsExist",
 			cachedItems: map[string]testInfo{
-				targetFileName: {
+				suite.targetFileName: {
 					info: deets,
 					err:  nil,
 				},
 			},
-			expectedLen: 1,
+			// 1 file and 5 folders.
+			expectedLen: 6,
 		},
 		{
 			name: "PendingNoDetails",
 			cachedItems: map[string]testInfo{
-				targetFileName: {
+				suite.targetFileName: {
 					info: nil,
 					err:  nil,
 				},
@@ -190,7 +194,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 		{
 			name: "HadError",
 			cachedItems: map[string]testInfo{
-				targetFileName: {
+				suite.targetFileName: {
 					info: deets,
 					err:  assert.AnError,
 				},
@@ -227,6 +231,65 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
 	}
 }
 
+func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchy() {
+	t := suite.T()
+	// Order of folders in hierarchy from root to leaf (excluding the item).
+	expectedFolderOrder := suite.targetFilePath.ToBuilder().Dir().Elements()
+
+	// Setup stuff.
+	bd := &details.Details{}
+	cp := corsoProgress{
+		UploadProgress: &snapshotfs.NullUploadProgress{},
+		deets:          bd,
+		pending:        map[string]*itemDetails{},
+	}
+
+	deets := &itemDetails{details.ItemInfo{}, suite.targetFilePath}
+	cp.put(suite.targetFileName, deets)
+	require.Len(t, cp.pending, 1)
+
+	cp.FinishedFile(suite.targetFileName, nil)
+
+	// Gather information about the current state.
+	var (
+		curRef     *details.DetailsEntry
+		refToEntry = map[string]*details.DetailsEntry{}
+	)
+
+	for i := 0; i < len(bd.Entries); i++ {
+		e := &bd.Entries[i]
+		if e.Folder == nil {
+			continue
+		}
+
+		refToEntry[e.ShortRef] = e
+
+		if e.Folder.DisplayName == expectedFolderOrder[len(expectedFolderOrder)-1] {
+			curRef = e
+		}
+	}
+
+	// Actual tests start here.
+	var rootRef *details.DetailsEntry
+
+	// Traverse the details entries from leaf to root, following the ParentRef
+	// fields. At the end rootRef should point to the root of the path.
+	for i := len(expectedFolderOrder) - 1; i >= 0; i-- {
+		name := expectedFolderOrder[i]
+
+		require.NotNil(t, curRef)
+		assert.Equal(t, name, curRef.Folder.DisplayName)
+
+		rootRef = curRef
+		curRef = refToEntry[curRef.ParentRef]
+	}
+
+	// Hierarchy root's ParentRef = "" and map will return nil.
+	assert.Nil(t, curRef)
+	require.NotNil(t, rootRef)
+	assert.Empty(t, rootRef.ParentRef)
+}
+
 type KopiaUnitSuite struct {
 	suite.Suite
 	testPath path.Path
@@ -595,7 +658,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
 	assert.Equal(t, stats.IgnoredErrorCount, 0)
 	assert.Equal(t, stats.ErrorCount, 0)
 	assert.False(t, stats.Incomplete)
-	assert.Len(t, rp.Entries, 47)
+	// 47 file and 6 folder entries.
+	assert.Len(t, rp.Entries, 47+6)
 }
 
 func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
@@ -690,7 +754,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
 	assert.Equal(t, 6, stats.TotalDirectoryCount)
 	assert.Equal(t, 1, stats.IgnoredErrorCount)
 	assert.False(t, stats.Incomplete)
-	assert.Len(t, rp.Entries, 5)
+	// 5 file and 6 folder entries.
+	assert.Len(t, rp.Entries, 5+6)
 }
 
 type KopiaSimpleRepoIntegrationSuite struct {
@@ -794,7 +859,8 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
 	require.Equal(t, stats.TotalDirectoryCount, 6)
 	require.Equal(t, stats.IgnoredErrorCount, 0)
 	require.False(t, stats.Incomplete)
-	assert.Len(t, rp.Entries, 6)
+	// 6 file and 6 folder entries.
+	assert.Len(t, rp.Entries, 6+6)
 
 	suite.snapshotID = manifest.ID(stats.SnapshotID)
 

src/internal/path/path.go

@@ -91,6 +91,8 @@ type Path interface {
 	// reference is guaranteed to be unique. No guarantees are made about whether
 	// a short reference can be converted back into the Path that generated it.
 	ShortRef() string
+	// ToBuilder returns a Builder instance that represents the current Path.
+	ToBuilder() *Builder
 }
 
 // Builder is a simple path representation that only tracks path elements. It
@@ -174,7 +176,7 @@ func (pb Builder) PopFront() *Builder {
 	}
 }
 
-func (pb Builder) dir() *Builder {
+func (pb Builder) Dir() *Builder {
 	if len(pb.elements) <= 1 {
 		return &Builder{}
 	}

src/internal/path/resource_path.go

@@ -171,7 +171,7 @@ func (rp dataLayerResourcePath) Dir() (Path, error) {
 	}
 
 	return &dataLayerResourcePath{
-		Builder:  *rp.dir(),
+		Builder:  *rp.Builder.Dir(),
 		service:  rp.service,
 		category: rp.category,
 		hasItem:  false,
@@ -193,3 +193,8 @@ func (rp dataLayerResourcePath) Append(
 		hasItem:  isItem,
 	}, nil
 }
+
+func (rp dataLayerResourcePath) ToBuilder() *Builder {
+	// Safe to directly return the Builder because Builders are immutable.
+	return &rp.Builder
+}

src/pkg/backup/details/details.go

@@ -14,6 +14,7 @@ import (
 type FolderEntry struct {
 	RepoRef   string
 	ShortRef  string
+	ParentRef string
 	Info      ItemInfo
 }
 
@@ -66,18 +67,39 @@ func printJSON(ctx context.Context, dm DetailsModel) {
 	print.All(ctx, ents...)
 }
 
-// Paths returns the list of Paths extracted from the Entries slice.
+// Paths returns the list of Paths for non-folder items extracted from the
+// Entries slice.
 func (dm DetailsModel) Paths() []string {
-	ents := dm.Entries
+	r := make([]string, 0, len(dm.Entries))
-	r := make([]string, len(ents))
 
-	for i := range ents {
+	for _, ent := range dm.Entries {
-		r[i] = ents[i].RepoRef
+		if ent.Folder != nil {
+			continue
+		}
+
+		r = append(r, ent.RepoRef)
 	}
 
 	return r
 }
 
+// Items returns a slice of *ItemInfo that does not contain any FolderInfo
+// entries. Required because not all folders in the details are valid resource
+// paths.
+func (dm DetailsModel) Items() []*DetailsEntry {
+	res := make([]*DetailsEntry, 0, len(dm.Entries))
+
+	for i := 0; i < len(dm.Entries); i++ {
+		if dm.Entries[i].Folder != nil {
+			continue
+		}
+
+		res = append(res, &dm.Entries[i])
+	}
+
+	return res
+}
+
 // --------------------------------------------------------------------------------
 // Details
 // --------------------------------------------------------------------------------
@@ -93,12 +115,13 @@ type Details struct {
 	knownFolders map[string]struct{} `json:"-"`
 }
 
-func (d *Details) Add(repoRef, shortRef string, info ItemInfo) {
+func (d *Details) Add(repoRef, shortRef, parentRef string, info ItemInfo) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	d.Entries = append(d.Entries, DetailsEntry{
 		RepoRef:   repoRef,
 		ShortRef:  shortRef,
+		ParentRef: parentRef,
 		ItemInfo:  info,
 	})
 }
@@ -123,6 +146,7 @@ func (d *Details) AddFolders(folders []FolderEntry) {
 		d.Entries = append(d.Entries, DetailsEntry{
 			RepoRef:   folder.RepoRef,
 			ShortRef:  folder.ShortRef,
+			ParentRef: folder.ParentRef,
 			ItemInfo:  folder.Info,
 		})
 	}
@@ -138,6 +162,7 @@ type DetailsEntry struct {
 	// This can be optimized.
 	RepoRef   string `json:"repoRef"`
 	ShortRef  string `json:"shortRef"`
+	ParentRef string `json:"parentRef,omitempty"`
 	ItemInfo
 }
 

src/pkg/backup/details/details_test.go

@@ -132,23 +132,22 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() {
 	}
 }
 
-func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
+var pathItemsTable = []struct {
-	table := []struct {
 	name       string
 	ents       []details.DetailsEntry
-		expect []string
+	expectRefs []string
 }{
 	{
 		name:       "nil entries",
 		ents:       nil,
-			expect: []string{},
+		expectRefs: []string{},
 	},
 	{
 		name: "single entry",
 		ents: []details.DetailsEntry{
 			{RepoRef: "abcde"},
 		},
-			expect: []string{"abcde"},
+		expectRefs: []string{"abcde"},
 	},
 	{
 		name: "multiple entries",
@@ -156,17 +155,54 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
 			{RepoRef: "abcde"},
 			{RepoRef: "12345"},
 		},
-			expect: []string{"abcde", "12345"},
+		expectRefs: []string{"abcde", "12345"},
+	},
+	{
+		name: "multiple entries with folder",
+		ents: []details.DetailsEntry{
+			{RepoRef: "abcde"},
+			{RepoRef: "12345"},
+			{
+				RepoRef: "deadbeef",
+				ItemInfo: details.ItemInfo{
+					Folder: &details.FolderInfo{
+						DisplayName: "test folder",
+					},
+				},
+			},
+		},
+		expectRefs: []string{"abcde", "12345"},
 	},
 }
-	for _, test := range table {
+
+func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
+	for _, test := range pathItemsTable {
 		suite.T().Run(test.name, func(t *testing.T) {
 			d := details.Details{
 				DetailsModel: details.DetailsModel{
 					Entries: test.ents,
 				},
 			}
-			assert.Equal(t, test.expect, d.Paths())
+			assert.Equal(t, test.expectRefs, d.Paths())
+		})
+	}
+}
+
+func (suite *DetailsUnitSuite) TestDetailsModel_Items() {
+	for _, test := range pathItemsTable {
+		suite.T().Run(test.name, func(t *testing.T) {
+			d := details.Details{
+				DetailsModel: details.DetailsModel{
+					Entries: test.ents,
+				},
+			}
+
+			ents := d.Items()
+			assert.Len(t, ents, len(test.expectRefs))
+
+			for _, e := range ents {
+				assert.Contains(t, test.expectRefs, e.RepoRef)
+			}
 		})
 	}
 }
@@ -183,10 +219,12 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 				{
 					RepoRef:   "rr1",
 					ShortRef:  "sr1",
+					ParentRef: "pr1",
 				},
 				{
 					RepoRef:   "rr2",
 					ShortRef:  "sr2",
+					ParentRef: "pr2",
 				},
 			},
 			expectedShortRefs: []string{"sr1", "sr2"},
@@ -197,18 +235,22 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
 				{
 					RepoRef:   "rr1",
 					ShortRef:  "sr1",
+					ParentRef: "pr1",
 				},
 				{
 					RepoRef:   "rr2",
 					ShortRef:  "sr2",
+					ParentRef: "pr2",
 				},
 				{
 					RepoRef:   "rr1",
 					ShortRef:  "sr1",
+					ParentRef: "pr1",
 				},
 				{
 					RepoRef:   "rr3",
 					ShortRef:  "sr3",
+					ParentRef: "pr3",
 				},
 			},
 			expectedShortRefs: []string{"sr1", "sr2", "sr3"},

src/pkg/selectors/scopes.go

@@ -227,7 +227,7 @@ func reduce[T scopeT, C categoryT](
 	ents := []details.DetailsEntry{}
 
 	// for each entry, compare that entry against the scopes of the same data type
-	for _, ent := range deets.Entries {
+	for _, ent := range deets.Items() {
 		repoPath, err := path.FromDataLayerPath(ent.RepoRef, true)
 		if err != nil {
 			logger.Ctx(ctx).Debugw("transforming repoRef to path", "err", err)
@@ -242,13 +242,13 @@ func reduce[T scopeT, C categoryT](
 		passed := passes(
 			dc,
 			dc.pathValues(repoPath),
-			ent,
+			*ent,
 			excls[dc],
 			filts[dc],
 			incls[dc],
 		)
 		if passed {
-			ents = append(ents, ent)
+			ents = append(ents, *ent)
 		}
 	}