larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break file order dependency for OneDrive .meta files (#2450)

Browse Source 
## Description

Begin using the Fetch() interface to retrieve OneDrive meta files inline when restoring the file. This removes the ordering dependency between .data and .meta files

This does not stop .meta files from being returned over the Items() channel. That can be disabled in a future PR

## Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No 

## Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

## Issue(s)

* #2447

## Test Plan

- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2023-02-09 10:30:04 -08:00 committed by GitHub
parent dad6776861
commit d9d0158b6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 205 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/internal/connector/graph_connector_helper_test.go
View File

@ -1,6 +1,7 @@
package connector
import (
"bytes"
"context"
"encoding/json"
"io"
@ -165,6 +166,10 @@ type colInfo struct {
pathElements []string
category path.CategoryType
items []itemInfo
// auxItems are items that can be retrieved with Fetch but won't be returned
// by Items(). These files do not directly participate in comparisosn at the
// end of a test.
auxItems []itemInfo
}
type restoreBackupInfo struct {
@ -969,6 +974,25 @@ func backupOutputPathFromRestore(
)
}
// TODO(ashmrtn): Make this an actual mock class that can be used in other
// packages.
type mockRestoreCollection struct {
data.Collection
auxItems map[string]data.Stream
}
func (rc mockRestoreCollection) Fetch(
ctx context.Context,
name string,
) (data.Stream, error) {
res := rc.auxItems[name]
if res == nil {
return nil, data.ErrNotFound
}
return res, nil
}
func collectionsForInfo(
t *testing.T,
service path.ServiceType,
@ -991,7 +1015,7 @@ func collectionsForInfo(
info.pathElements,
false,
)
c := mockconnector.NewMockExchangeCollection(pth, len(info.items))
mc := mockconnector.NewMockExchangeCollection(pth, len(info.items))
baseDestPath := backupOutputPathFromRestore(t, dest, pth)
baseExpected := expectedData[baseDestPath.String()]
@ -1001,8 +1025,8 @@ func collectionsForInfo(
}
for i := 0; i < len(info.items); i++ {
c.Names[i] = info.items[i].name
c.Data[i] = info.items[i].data
mc.Names[i] = info.items[i].name
mc.Data[i] = info.items[i].data
baseExpected[info.items[i].lookupKey] = info.items[i].data
@ -1014,9 +1038,16 @@ func collectionsForInfo(
}
}
collections = append(collections, data.NotFoundRestoreCollection{
Collection: c,
})
c := mockRestoreCollection{Collection: mc, auxItems: map[string]data.Stream{}}
for _, aux := range info.auxItems {
c.auxItems[aux.name] = &mockconnector.MockExchangeData{
ID: aux.name,
Reader: io.NopCloser(bytes.NewReader(aux.data)),
}
}
collections = append(collections, c)
kopiaEntries += len(info.items)
}

140
src/internal/connector/graph_connector_test.go
View File

@ -898,6 +898,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -924,6 +931,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -951,6 +965,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "folder-a" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -974,6 +995,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -995,6 +1023,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1027,6 +1062,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"write"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1048,6 +1090,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"read"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1203,6 +1252,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackupVersion0() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1229,6 +1285,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackupVersion0() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1256,6 +1319,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackupVersion0() {
lookupKey: "folder-a" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1279,6 +1349,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackupVersion0() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1300,6 +1377,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackupVersion0() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1521,6 +1605,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"write"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1554,6 +1645,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1575,6 +1673,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"read"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1608,6 +1713,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "b" + onedrive.DirMetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"write"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
{
pathElements: []string{
@ -1629,6 +1741,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"read"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1673,6 +1792,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"write"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1717,6 +1843,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsRestoreAndBackup() {
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: []byte("{}"),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},
@ -1775,6 +1908,13 @@ func (suite *GraphConnectorIntegrationSuite) TestPermissionsBackupAndNoRestore()
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
auxItems: []itemInfo{
{
name: "test-file.txt" + onedrive.MetaFileSuffix,
data: getTestMetaJSON(suite.T(), suite.secondaryUser, []string{"write"}),
lookupKey: "test-file.txt" + onedrive.MetaFileSuffix,
},
},
},
},
},

48
src/internal/connector/onedrive/restore.go
View File

@ -9,6 +9,7 @@ import (
"sort"
"strings"
"github.com/alcionai/clues"
msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
@ -164,7 +165,6 @@ func RestoreCollection(
metrics = support.CollectionMetrics{}
copyBuffer = make([]byte, copyBufferSize)
directory = dc.FullPath()
restoredIDs = map[string]string{}
itemInfo details.ItemInfo
itemID string
folderPerms = map[string][]UserPermission{}
@ -226,37 +226,44 @@ func RestoreCollection(
metrics.TotalBytes += int64(len(copyBuffer))
trimmedName := strings.TrimSuffix(name, DataFileSuffix)
itemID, itemInfo, err = restoreData(ctx, service, trimmedName, itemData,
drivePath.DriveID, restoreFolderID, copyBuffer, source)
itemID, itemInfo, err = restoreData(
ctx,
service,
trimmedName,
itemData,
drivePath.DriveID,
restoreFolderID,
copyBuffer,
source)
if err != nil {
errUpdater(itemData.UUID(), err)
continue
}
restoredIDs[trimmedName] = itemID
deets.Add(itemPath.String(), itemPath.ShortRef(), "", true, itemInfo)
// Mark it as success without processing .meta
// file if we are not restoring permissions
if !restorePerms {
metrics.Successes++
}
} else if strings.HasSuffix(name, MetaFileSuffix) {
if !restorePerms {
continue
}
meta, err := getMetadata(itemData.ToReader())
// Fetch item permissions from the collection and restore them.
metaName := trimmedName + MetaFileSuffix
permsFile, err := dc.Fetch(ctx, metaName)
if err != nil {
errUpdater(itemData.UUID(), err)
errUpdater(metaName, clues.Wrap(err, "getting item metadata"))
continue
}
trimmedName := strings.TrimSuffix(name, MetaFileSuffix)
restoreID, ok := restoredIDs[trimmedName]
if !ok {
errUpdater(itemData.UUID(), fmt.Errorf("item not available to restore permissions"))
metaReader := permsFile.ToReader()
meta, err := getMetadata(metaReader)
metaReader.Close()
if err != nil {
errUpdater(metaName, clues.Wrap(err, "deserializing item metadata"))
continue
}
@ -264,21 +271,22 @@ func RestoreCollection(
ctx,
service,
drivePath.DriveID,
restoreID,
itemID,
parentPerms,
meta.Permissions,
permissionIDMappings,
)
if err != nil {
errUpdater(itemData.UUID(), err)
errUpdater(trimmedName, clues.Wrap(err, "restoring item permissions"))
continue
}
// Objects count is incremented when we restore a
// data file and success count is incremented when
// we restore a meta file as every data file
// should have an associated meta file
metrics.Successes++
} else if strings.HasSuffix(name, MetaFileSuffix) {
// Just skip this for the moment since we moved the code to the above
// item restore path. We haven't yet stopped fetching these items in
// RestoreOp, so we still need to handle them in some way.
continue
} else if strings.HasSuffix(name, DirMetaFileSuffix) {
trimmedName := strings.TrimSuffix(name, DirMetaFileSuffix)
folderID, err := createRestoreFolder(