larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update folder size and modified time in details (#1881)

Browse Source 
## Description

Caches folder info added during details construction in the details builder and keeps the size/modified time
updated as newer items are added.

As part of this, this PR refactors the details package to separate out building `details.Details` from the in-memory representation and model.

## Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #1850 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Vaibhav Kamra 2022-12-21 11:17:24 -08:00 committed by GitHub
parent b59e5849ab
commit a45aeda4a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 239 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/internal/connector/exchange/service_restore.go
View File

@ -288,7 +288,7 @@ func RestoreExchangeDataCollections(
gs graph.Servicer,
dest control.RestoreDestination,
dcs []data.Collection,
deets *details.Details,
deets *details.Builder,
) (*support.ConnectorOperationStatus, error) {
var (
// map of caches... but not yet...
@ -349,7 +349,7 @@ func restoreCollection(
dc data.Collection,
folderID string,
policy control.CollisionPolicy,
deets *details.Details,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:exchange:restoreCollection", D.Label("path", dc.FullPath()))

4
src/internal/connector/graph_connector.go
View File

@ -262,7 +262,7 @@ func (gc *GraphConnector) RestoreDataCollections(
var (
status *support.ConnectorOperationStatus
err error
deets = &details.Details{}
deets = &details.Builder{}
)
switch selector.Service {
@ -279,7 +279,7 @@ func (gc *GraphConnector) RestoreDataCollections(
gc.incrementAwaitingMessages()
gc.UpdateStatus(status)
return deets, err
return deets.Details(), err
}
// AwaitStatus waits for all gc tasks to complete and then returns status

4
src/internal/connector/onedrive/restore.go
View File

@ -53,7 +53,7 @@ func RestoreCollections(
service graph.Servicer,
dest control.RestoreDestination,
dcs []data.Collection,
deets *details.Details,
deets *details.Builder,
) (*support.ConnectorOperationStatus, error) {
var (
restoreMetrics support.CollectionMetrics
@ -95,7 +95,7 @@ func RestoreCollection(
dc data.Collection,
source driveSource,
restoreContainerName string,
deets *details.Details,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:oneDrive:restoreCollection", D.Label("path", dc.FullPath()))

2
src/internal/connector/sharepoint/restore.go
View File

@ -20,7 +20,7 @@ func RestoreCollections(
service graph.Servicer,
dest control.RestoreDestination,
dcs []data.Collection,
deets *details.Details,
deets *details.Builder,
) (*support.ConnectorOperationStatus, error) {
var (
restoreMetrics support.CollectionMetrics

4
src/internal/kopia/upload.go
View File

@ -125,7 +125,7 @@ type itemDetails struct {
type corsoProgress struct {
snapshotfs.UploadProgress
pending map[string]*itemDetails
deets *details.Details
deets *details.Builder
mu sync.RWMutex
totalBytes int64
}
@ -182,7 +182,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
parent = nextParent
}
cp.deets.AddFolders(folders)
cp.deets.AddFoldersForItem(folders, d.info)
}
// Kopia interface function used as a callback when kopia finishes hashing a file.

14
src/internal/kopia/upload_test.go
View File

@ -435,7 +435,7 @@ var finishedFileTable = []struct {
func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
for _, test := range finishedFileTable {
suite.T().Run(test.name, func(t *testing.T) {
bd := &details.Details{}
bd := &details.Builder{}
cp := corsoProgress{
UploadProgress: &snapshotfs.NullUploadProgress{},
deets: bd,
@ -455,7 +455,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
}
assert.Empty(t, cp.pending)
assert.Len(t, bd.Entries, test.expectedNumEntries)
assert.Len(t, bd.Details().Entries, test.expectedNumEntries)
})
}
}
@ -466,7 +466,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchy() {
expectedFolderOrder := suite.targetFilePath.ToBuilder().Dir().Elements()
// Setup stuff.
bd := &details.Details{}
bd := &details.Builder{}
cp := corsoProgress{
UploadProgress: &snapshotfs.NullUploadProgress{},
deets: bd,
@ -485,8 +485,10 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchy() {
refToEntry = map[string]*details.DetailsEntry{}
)
for i := 0; i < len(bd.Entries); i++ {
e := &bd.Entries[i]
entries := bd.Details().Entries
for i := 0; i < len(entries); i++ {
e := &entries[i]
if e.Folder == nil {
continue
}
@ -522,7 +524,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileBuildsHierarchy() {
func (suite *CorsoProgressUnitSuite) TestFinishedHashingFile() {
for _, test := range finishedFileTable {
suite.T().Run(test.name, func(t *testing.T) {
bd := &details.Details{}
bd := &details.Builder{}
cp := corsoProgress{
UploadProgress: &snapshotfs.NullUploadProgress{},
deets: bd,

6
src/internal/kopia/wrapper.go
View File

@ -131,12 +131,12 @@ func (w Wrapper) BackupCollections(
defer end()
if len(collections) == 0 {
return &BackupStats{}, &details.Details{}, nil
return &BackupStats{}, (&details.Builder{}).Details(), nil
}
progress := &corsoProgress{
pending: map[string]*itemDetails{},
deets: &details.Details{},
deets: &details.Builder{},
}
// TODO(ashmrtn): Pass previousSnapshots here to enable building the directory
@ -158,7 +158,7 @@ func (w Wrapper) BackupCollections(
return nil, nil, err
}
return s, progress.deets, nil
return s, progress.deets.Details(), nil
}
func (w Wrapper) makeSnapshotWithRoot(

6
src/internal/streamstore/streamstore_test.go
View File

@ -46,15 +46,17 @@ func (suite *StreamStoreIntegrationSuite) TestDetails() {
defer kw.Close(ctx)
deets := &details.Details{}
deetsBuilder := &details.Builder{}
deets.Add("ref", "shortref", "parentref", true,
deetsBuilder.Add("ref", "shortref", "parentref", true,
details.ItemInfo{
Exchange: &details.ExchangeInfo{
Subject: "hello world",
},
})
deets := deetsBuilder.Details()
ss := New(kw, "tenant", path.ExchangeService)
id, err := ss.WriteBackupDetails(ctx, deets)

112
src/pkg/backup/details/details.go
View File

@ -100,6 +100,78 @@ func (dm DetailsModel) Items() []*DetailsEntry {
return res
}
// Builder should be used to create a details model.
type Builder struct {
d Details
mu sync.Mutex `json:"-"`
knownFolders map[string]FolderEntry `json:"-"`
}
func (b *Builder) Add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
b.mu.Lock()
defer b.mu.Unlock()
b.d.add(repoRef, shortRef, parentRef, updated, info)
}
func (b *Builder) Details() *Details {
b.mu.Lock()
defer b.mu.Unlock()
// Write the cached folder entries to details
for _, folder := range b.knownFolders {
b.d.addFolder(folder)
}
return &b.d
}
// AddFoldersForItem adds entries for the given folders. It skips adding entries that
// have been added by previous calls.
func (b *Builder) AddFoldersForItem(folders []FolderEntry, itemInfo ItemInfo) {
b.mu.Lock()
defer b.mu.Unlock()
if b.knownFolders == nil {
b.knownFolders = map[string]FolderEntry{}
}
for _, folder := range folders {
if existing, ok := b.knownFolders[folder.ShortRef]; ok {
// We've seen this folder before for a different item.
// Update the "cached" folder entry
folder = existing
}
// Update the folder's size and modified time
var (
itemSize int64
itemModified time.Time
)
switch {
case itemInfo.Exchange != nil:
itemSize = itemInfo.Exchange.Size
itemModified = itemInfo.Exchange.Modified
case itemInfo.OneDrive != nil:
itemSize = itemInfo.OneDrive.Size
itemModified = itemInfo.OneDrive.Modified
case itemInfo.SharePoint != nil:
itemSize = itemInfo.SharePoint.Size
itemModified = itemInfo.SharePoint.Modified
}
folder.Info.Folder.Size += itemSize
if folder.Info.Folder.Modified.Before(itemModified) {
folder.Info.Folder.Modified = itemModified
}
b.knownFolders[folder.ShortRef] = folder
}
}
// --------------------------------------------------------------------------------
// Details
// --------------------------------------------------------------------------------
@ -109,15 +181,9 @@ func (dm DetailsModel) Items() []*DetailsEntry {
// printing.
type Details struct {
DetailsModel
// internal
mu sync.Mutex `json:"-"`
knownFolders map[string]struct{} `json:"-"`
}
func (d *Details) Add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
d.mu.Lock()
defer d.mu.Unlock()
func (d *Details) add(repoRef, shortRef, parentRef string, updated bool, info ItemInfo) {
d.Entries = append(d.Entries, DetailsEntry{
RepoRef: repoRef,
ShortRef: shortRef,
@ -127,30 +193,14 @@ func (d *Details) Add(repoRef, shortRef, parentRef string, updated bool, info It
})
}
// AddFolders adds entries for the given folders. It skips adding entries that
// have been added by previous calls.
func (d *Details) AddFolders(folders []FolderEntry) {
d.mu.Lock()
defer d.mu.Unlock()
if d.knownFolders == nil {
d.knownFolders = map[string]struct{}{}
}
for _, folder := range folders {
if _, ok := d.knownFolders[folder.ShortRef]; ok {
// Entry already exists, nothing to do.
continue
}
d.knownFolders[folder.ShortRef] = struct{}{}
d.Entries = append(d.Entries, DetailsEntry{
RepoRef: folder.RepoRef,
ShortRef: folder.ShortRef,
ParentRef: folder.ParentRef,
ItemInfo: folder.Info,
})
}
// addFolder adds an entry for the given folder.
func (d *Details) addFolder(folder FolderEntry) {
d.Entries = append(d.Entries, DetailsEntry{
RepoRef: folder.RepoRef,
ShortRef: folder.ShortRef,
ParentRef: folder.ParentRef,
ItemInfo: folder.Info,
})
}
// --------------------------------------------------------------------------------

140
src/pkg/backup/details/details_test.go
View File

@ -222,10 +222,22 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Items() {
}
func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
itemTime := time.Date(2022, 10, 21, 10, 0, 0, 0, time.UTC)
folderTimeOlderThanItem := time.Date(2022, 9, 21, 10, 0, 0, 0, time.UTC)
folderTimeNewerThanItem := time.Date(2022, 11, 21, 10, 0, 0, 0, time.UTC)
itemInfo := details.ItemInfo{
Exchange: &details.ExchangeInfo{
Size: 20,
Modified: itemTime,
},
}
table := []struct {
name string
folders []details.FolderEntry
expectedShortRefs []string
name string
folders []details.FolderEntry
expectedShortRefs []string
expectedFolderInfo map[string]details.FolderInfo
}{
{
name: "MultipleFolders",
@ -234,14 +246,28 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeOlderThanItem,
},
},
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeNewerThanItem,
},
},
},
},
expectedShortRefs: []string{"sr1", "sr2"},
expectedFolderInfo: map[string]details.FolderInfo{
"sr1": {Size: 20, Modified: itemTime},
"sr2": {Size: 20, Modified: folderTimeNewerThanItem},
},
},
{
name: "MultipleFoldersWithRepeats",
@ -250,36 +276,138 @@ func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeOlderThanItem,
},
},
},
{
RepoRef: "rr2",
ShortRef: "sr2",
ParentRef: "pr2",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeOlderThanItem,
},
},
},
{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeOlderThanItem,
},
},
},
{
RepoRef: "rr3",
ShortRef: "sr3",
ParentRef: "pr3",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeNewerThanItem,
},
},
},
},
expectedShortRefs: []string{"sr1", "sr2", "sr3"},
expectedFolderInfo: map[string]details.FolderInfo{
// Two items were added
"sr1": {Size: 40, Modified: itemTime},
"sr2": {Size: 20, Modified: itemTime},
"sr3": {Size: 20, Modified: folderTimeNewerThanItem},
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
deets := details.Details{}
deets.AddFolders(test.folders)
builder := details.Builder{}
builder.AddFoldersForItem(test.folders, itemInfo)
deets := builder.Details()
assert.Len(t, deets.Entries, len(test.expectedShortRefs))
for _, e := range deets.Entries {
assert.Contains(t, test.expectedShortRefs, e.ShortRef)
assert.Equal(t, test.expectedFolderInfo[e.ShortRef].Size, e.Folder.Size)
assert.Equal(t, test.expectedFolderInfo[e.ShortRef].Modified, e.Folder.Modified)
}
})
}
}
func (suite *DetailsUnitSuite) TestDetails_AddFoldersDifferentServices() {
itemTime := time.Date(2022, 10, 21, 10, 0, 0, 0, time.UTC)
folderTimeOlderThanItem := time.Date(2022, 9, 21, 10, 0, 0, 0, time.UTC)
table := []struct {
name string
item details.ItemInfo
expectedFolderInfo details.FolderInfo
}{
{
name: "Exchange",
item: details.ItemInfo{
Exchange: &details.ExchangeInfo{
Size: 20,
Modified: itemTime,
},
},
expectedFolderInfo: details.FolderInfo{
Size: 20,
Modified: itemTime,
},
},
{
name: "OneDrive",
item: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
Size: 20,
Modified: itemTime,
},
},
expectedFolderInfo: details.FolderInfo{
Size: 20,
Modified: itemTime,
},
},
{
name: "SharePoint",
item: details.ItemInfo{
SharePoint: &details.SharePointInfo{
Size: 20,
Modified: itemTime,
},
},
expectedFolderInfo: details.FolderInfo{
Size: 20,
Modified: itemTime,
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
folderEntry := details.FolderEntry{
RepoRef: "rr1",
ShortRef: "sr1",
ParentRef: "pr1",
Info: details.ItemInfo{
Folder: &details.FolderInfo{
Modified: folderTimeOlderThanItem,
},
},
}
builder := details.Builder{}
builder.AddFoldersForItem([]details.FolderEntry{folderEntry}, test.item)
deets := builder.Details()
require.Len(t, deets.Entries, 1)
got := deets.Entries[0].Folder
assert.Equal(t, test.expectedFolderInfo, *got)
})
}
}