don't handle meta files in details (#2606)
## Description Remove the display and interaction with .meta files from details entries. ## Does this PR need a docs update or release note? - [x] ✅ Yes, it's included ## Type of change - [x] 🐛 Bugfix ## Test Plan - [x] ⚡ Unit test
This commit is contained in:
parent
6c7623041c
commit
cb12329d0c
@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Added
|
### Added
|
||||||
- Show owner information when doing backup list in json format
|
- Show owner information when doing backup list in json format
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Corso-generated .meta files and permissions no longer appear in the backup details.
|
||||||
|
|
||||||
### Known Issues
|
### Known Issues
|
||||||
- Folders and Calendars containing zero items or subfolders are not included in the backup.
|
- Folders and Calendars containing zero items or subfolders are not included in the backup.
|
||||||
|
|
||||||
|
|||||||
@ -15,8 +15,8 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
"github.com/alcionai/corso/src/internal/connector/mockconnector"
|
"github.com/alcionai/corso/src/internal/connector/mockconnector"
|
||||||
"github.com/alcionai/corso/src/internal/data"
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/credentials"
|
"github.com/alcionai/corso/src/pkg/credentials"
|
||||||
@ -94,7 +94,7 @@ func generateAndRestoreItems(
|
|||||||
|
|
||||||
Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
|
Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
|
||||||
|
|
||||||
return gc.RestoreDataCollections(ctx, backup.Version, acct, sel, dest, opts, dataColls, errs)
|
return gc.RestoreDataCollections(ctx, version.Backup, acct, sel, dest, opts, dataColls, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------------------------------
|
// ------------------------------------------------------------------------------------------
|
||||||
|
|||||||
@ -712,6 +712,7 @@ func compareOneDriveItem(
|
|||||||
}
|
}
|
||||||
|
|
||||||
name := item.UUID()
|
name := item.UUID()
|
||||||
|
|
||||||
if strings.HasSuffix(name, onedrive.MetaFileSuffix) ||
|
if strings.HasSuffix(name, onedrive.MetaFileSuffix) ||
|
||||||
strings.HasSuffix(name, onedrive.DirMetaFileSuffix) {
|
strings.HasSuffix(name, onedrive.DirMetaFileSuffix) {
|
||||||
var (
|
var (
|
||||||
@ -1050,10 +1051,12 @@ func collectionsForInfo(
|
|||||||
allInfo []colInfo,
|
allInfo []colInfo,
|
||||||
backupVersion int,
|
backupVersion int,
|
||||||
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
|
) (int, int, []data.RestoreCollection, map[string]map[string][]byte) {
|
||||||
collections := make([]data.RestoreCollection, 0, len(allInfo))
|
var (
|
||||||
expectedData := make(map[string]map[string][]byte, len(allInfo))
|
collections = make([]data.RestoreCollection, 0, len(allInfo))
|
||||||
totalItems := 0
|
expectedData = make(map[string]map[string][]byte, len(allInfo))
|
||||||
kopiaEntries := 0
|
totalItems = 0
|
||||||
|
kopiaEntries = 0
|
||||||
|
)
|
||||||
|
|
||||||
for _, info := range allInfo {
|
for _, info := range allInfo {
|
||||||
pth := mustToDataLayerPath(
|
pth := mustToDataLayerPath(
|
||||||
|
|||||||
@ -14,8 +14,8 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
@ -175,9 +175,7 @@ func (c *onedriveCollection) withFile(
|
|||||||
name+onedrive.DataFileSuffix,
|
name+onedrive.DataFileSuffix,
|
||||||
fileData))
|
fileData))
|
||||||
|
|
||||||
case 1:
|
case 1, 2, 3:
|
||||||
fallthrough
|
|
||||||
case 2:
|
|
||||||
c.items = append(c.items, onedriveItemWithData(
|
c.items = append(c.items, onedriveItemWithData(
|
||||||
c.t,
|
c.t,
|
||||||
name+onedrive.DataFileSuffix,
|
name+onedrive.DataFileSuffix,
|
||||||
@ -209,9 +207,7 @@ func (c *onedriveCollection) withFolder(
|
|||||||
case 0:
|
case 0:
|
||||||
return c
|
return c
|
||||||
|
|
||||||
case 1:
|
case 1, 2, 3:
|
||||||
fallthrough
|
|
||||||
case 2:
|
|
||||||
c.items = append(
|
c.items = append(
|
||||||
c.items,
|
c.items,
|
||||||
onedriveMetadata(
|
onedriveMetadata(
|
||||||
@ -219,8 +215,7 @@ func (c *onedriveCollection) withFolder(
|
|||||||
"",
|
"",
|
||||||
name+onedrive.DirMetaFileSuffix,
|
name+onedrive.DirMetaFileSuffix,
|
||||||
user,
|
user,
|
||||||
roles),
|
roles))
|
||||||
)
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
|
assert.FailNowf(c.t, "bad backup version", "version %d", c.backupVersion)
|
||||||
@ -237,7 +232,7 @@ func (c *onedriveCollection) withPermissions(
|
|||||||
) *onedriveCollection {
|
) *onedriveCollection {
|
||||||
// These versions didn't store permissions for the folder or didn't store them
|
// These versions didn't store permissions for the folder or didn't store them
|
||||||
// in the folder's collection.
|
// in the folder's collection.
|
||||||
if c.backupVersion < 3 {
|
if c.backupVersion < version.OneDriveXIncludesPermissions {
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -281,7 +276,7 @@ type onedriveColInfo struct {
|
|||||||
type onedriveTest struct {
|
type onedriveTest struct {
|
||||||
name string
|
name string
|
||||||
// Version this test first be run for. Will run from
|
// Version this test first be run for. Will run from
|
||||||
// [startVersion, backup.Version] inclusive.
|
// [startVersion, version.Backup] inclusive.
|
||||||
startVersion int
|
startVersion int
|
||||||
cols []onedriveColInfo
|
cols []onedriveColInfo
|
||||||
}
|
}
|
||||||
@ -467,17 +462,17 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestRestoreAndBackup_Multip
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range table {
|
for _, test := range table {
|
||||||
expected := testDataForInfo(suite.T(), test.cols, backup.Version)
|
expected := testDataForInfo(suite.T(), test.cols, version.Backup)
|
||||||
|
|
||||||
for version := test.startVersion; version <= backup.Version; version++ {
|
for vn := test.startVersion; vn <= version.Backup; vn++ {
|
||||||
suite.T().Run(fmt.Sprintf("%s_Version%d", test.name, version), func(t *testing.T) {
|
suite.T().Run(fmt.Sprintf("%s_Version%d", test.name, vn), func(t *testing.T) {
|
||||||
input := testDataForInfo(t, test.cols, version)
|
input := testDataForInfo(t, test.cols, vn)
|
||||||
|
|
||||||
testData := restoreBackupInfoMultiVersion{
|
testData := restoreBackupInfoMultiVersion{
|
||||||
service: path.OneDriveService,
|
service: path.OneDriveService,
|
||||||
resource: Users,
|
resource: Users,
|
||||||
backupVersion: version,
|
backupVersion: vn,
|
||||||
countMeta: version == 0,
|
countMeta: vn == 0,
|
||||||
collectionsPrevious: input,
|
collectionsPrevious: input,
|
||||||
collectionsLatest: expected,
|
collectionsLatest: expected,
|
||||||
}
|
}
|
||||||
@ -691,17 +686,17 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndBa
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range table {
|
for _, test := range table {
|
||||||
expected := testDataForInfo(suite.T(), test.cols, backup.Version)
|
expected := testDataForInfo(suite.T(), test.cols, version.Backup)
|
||||||
|
|
||||||
for version := test.startVersion; version <= backup.Version; version++ {
|
for vn := test.startVersion; vn <= version.Backup; vn++ {
|
||||||
suite.T().Run(fmt.Sprintf("%s_Version%d", test.name, version), func(t *testing.T) {
|
suite.T().Run(fmt.Sprintf("%s_Version%d", test.name, vn), func(t *testing.T) {
|
||||||
input := testDataForInfo(t, test.cols, version)
|
input := testDataForInfo(t, test.cols, vn)
|
||||||
|
|
||||||
testData := restoreBackupInfoMultiVersion{
|
testData := restoreBackupInfoMultiVersion{
|
||||||
service: path.OneDriveService,
|
service: path.OneDriveService,
|
||||||
resource: Users,
|
resource: Users,
|
||||||
backupVersion: version,
|
backupVersion: vn,
|
||||||
countMeta: version == 0,
|
countMeta: vn == 0,
|
||||||
collectionsPrevious: input,
|
collectionsPrevious: input,
|
||||||
collectionsLatest: expected,
|
collectionsLatest: expected,
|
||||||
}
|
}
|
||||||
@ -764,17 +759,17 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsBackupAndNoR
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range table {
|
for _, test := range table {
|
||||||
expected := testDataForInfo(suite.T(), test.cols, backup.Version)
|
expected := testDataForInfo(suite.T(), test.cols, version.Backup)
|
||||||
|
|
||||||
for version := test.startVersion; version <= backup.Version; version++ {
|
for vn := test.startVersion; vn <= version.Backup; vn++ {
|
||||||
suite.T().Run(fmt.Sprintf("Version%d", version), func(t *testing.T) {
|
suite.T().Run(fmt.Sprintf("Version%d", vn), func(t *testing.T) {
|
||||||
input := testDataForInfo(t, test.cols, version)
|
input := testDataForInfo(t, test.cols, vn)
|
||||||
|
|
||||||
testData := restoreBackupInfoMultiVersion{
|
testData := restoreBackupInfoMultiVersion{
|
||||||
service: path.OneDriveService,
|
service: path.OneDriveService,
|
||||||
resource: Users,
|
resource: Users,
|
||||||
backupVersion: version,
|
backupVersion: vn,
|
||||||
countMeta: version == 0,
|
countMeta: vn == 0,
|
||||||
collectionsPrevious: input,
|
collectionsPrevious: input,
|
||||||
collectionsLatest: expected,
|
collectionsLatest: expected,
|
||||||
}
|
}
|
||||||
@ -811,7 +806,7 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndNo
|
|||||||
test := restoreBackupInfoMultiVersion{
|
test := restoreBackupInfoMultiVersion{
|
||||||
service: path.OneDriveService,
|
service: path.OneDriveService,
|
||||||
resource: Users,
|
resource: Users,
|
||||||
backupVersion: backup.Version,
|
backupVersion: version.Backup,
|
||||||
countMeta: false,
|
countMeta: false,
|
||||||
collectionsPrevious: []colInfo{
|
collectionsPrevious: []colInfo{
|
||||||
newOneDriveCollection(
|
newOneDriveCollection(
|
||||||
@ -821,7 +816,7 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndNo
|
|||||||
driveID,
|
driveID,
|
||||||
"root:",
|
"root:",
|
||||||
},
|
},
|
||||||
backup.Version,
|
version.Backup,
|
||||||
).
|
).
|
||||||
withFile(
|
withFile(
|
||||||
fileName,
|
fileName,
|
||||||
@ -843,7 +838,7 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndNo
|
|||||||
"root:",
|
"root:",
|
||||||
folderBName,
|
folderBName,
|
||||||
},
|
},
|
||||||
backup.Version,
|
version.Backup,
|
||||||
).
|
).
|
||||||
withFile(
|
withFile(
|
||||||
fileName,
|
fileName,
|
||||||
@ -865,7 +860,7 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndNo
|
|||||||
driveID,
|
driveID,
|
||||||
"root:",
|
"root:",
|
||||||
},
|
},
|
||||||
backup.Version,
|
version.Backup,
|
||||||
).
|
).
|
||||||
withFile(
|
withFile(
|
||||||
fileName,
|
fileName,
|
||||||
@ -887,7 +882,7 @@ func (suite *GraphConnectorOneDriveIntegrationSuite) TestPermissionsRestoreAndNo
|
|||||||
"root:",
|
"root:",
|
||||||
folderBName,
|
folderBName,
|
||||||
},
|
},
|
||||||
backup.Version,
|
version.Backup,
|
||||||
).
|
).
|
||||||
withFile(
|
withFile(
|
||||||
fileName,
|
fileName,
|
||||||
|
|||||||
@ -17,8 +17,8 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
"github.com/alcionai/corso/src/internal/data"
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
@ -237,7 +237,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
|
|||||||
|
|
||||||
deets, err := suite.connector.RestoreDataCollections(
|
deets, err := suite.connector.RestoreDataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
backup.Version,
|
version.Backup,
|
||||||
acct,
|
acct,
|
||||||
sel,
|
sel,
|
||||||
dest,
|
dest,
|
||||||
@ -314,7 +314,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
|
|||||||
|
|
||||||
deets, err := suite.connector.RestoreDataCollections(
|
deets, err := suite.connector.RestoreDataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
backup.Version,
|
version.Backup,
|
||||||
suite.acct,
|
suite.acct,
|
||||||
test.sel,
|
test.sel,
|
||||||
dest,
|
dest,
|
||||||
@ -527,13 +527,13 @@ func runRestoreBackupTest(
|
|||||||
t,
|
t,
|
||||||
config,
|
config,
|
||||||
test.collections,
|
test.collections,
|
||||||
backup.Version)
|
version.Backup)
|
||||||
|
|
||||||
runRestore(
|
runRestore(
|
||||||
t,
|
t,
|
||||||
ctx,
|
ctx,
|
||||||
config,
|
config,
|
||||||
backup.Version,
|
version.Backup,
|
||||||
collections,
|
collections,
|
||||||
totalItems)
|
totalItems)
|
||||||
|
|
||||||
@ -590,7 +590,7 @@ func runRestoreBackupTestVersions(
|
|||||||
t,
|
t,
|
||||||
config,
|
config,
|
||||||
test.collectionsLatest,
|
test.collectionsLatest,
|
||||||
backup.Version)
|
version.Backup)
|
||||||
|
|
||||||
runBackupAndCompare(
|
runBackupAndCompare(
|
||||||
t,
|
t,
|
||||||
@ -930,7 +930,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
|||||||
suite.user,
|
suite.user,
|
||||||
dest,
|
dest,
|
||||||
[]colInfo{collection},
|
[]colInfo{collection},
|
||||||
backup.Version,
|
version.Backup,
|
||||||
)
|
)
|
||||||
allItems += totalItems
|
allItems += totalItems
|
||||||
|
|
||||||
@ -948,7 +948,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
|
|||||||
restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource)
|
restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource)
|
||||||
deets, err := restoreGC.RestoreDataCollections(
|
deets, err := restoreGC.RestoreDataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
backup.Version,
|
version.Backup,
|
||||||
suite.acct,
|
suite.acct,
|
||||||
restoreSel,
|
restoreSel,
|
||||||
dest,
|
dest,
|
||||||
|
|||||||
@ -406,7 +406,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
|
|||||||
|
|
||||||
// TODO(meain): Remove this once we change to always
|
// TODO(meain): Remove this once we change to always
|
||||||
// backing up permissions. Until then we cannot rely
|
// backing up permissions. Until then we cannot rely
|
||||||
// on weather the previous data is what we need as the
|
// on whether the previous data is what we need as the
|
||||||
// user might have not backup up permissions in the
|
// user might have not backup up permissions in the
|
||||||
// previous run.
|
// previous run.
|
||||||
metaItemInfo := details.ItemInfo{}
|
metaItemInfo := details.ItemInfo{}
|
||||||
@ -415,6 +415,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
|
|||||||
ItemName: itemInfo.OneDrive.ItemName,
|
ItemName: itemInfo.OneDrive.ItemName,
|
||||||
DriveName: itemInfo.OneDrive.DriveName,
|
DriveName: itemInfo.OneDrive.DriveName,
|
||||||
ItemType: itemInfo.OneDrive.ItemType,
|
ItemType: itemInfo.OneDrive.ItemType,
|
||||||
|
IsMeta: true,
|
||||||
Modified: time.Now(), // set to current time to always refresh
|
Modified: time.Now(), // set to current time to always refresh
|
||||||
Owner: itemInfo.OneDrive.Owner,
|
Owner: itemInfo.OneDrive.Owner,
|
||||||
ParentPath: itemInfo.OneDrive.ParentPath,
|
ParentPath: itemInfo.OneDrive.ParentPath,
|
||||||
|
|||||||
@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
|
||||||
"runtime/trace"
|
"runtime/trace"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@ -20,6 +19,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/data"
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
D "github.com/alcionai/corso/src/internal/diagnostics"
|
D "github.com/alcionai/corso/src/internal/diagnostics"
|
||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
@ -27,23 +27,10 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
// copyBufferSize is used for chunked upload
|
||||||
// copyBufferSize is used for chunked upload
|
// Microsoft recommends 5-10MB buffers
|
||||||
// Microsoft recommends 5-10MB buffers
|
// https://docs.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#best-practices
|
||||||
// https://docs.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#best-practices
|
const copyBufferSize = 5 * 1024 * 1024
|
||||||
copyBufferSize = 5 * 1024 * 1024
|
|
||||||
|
|
||||||
// versionWithDataAndMetaFiles is the corso backup format version
|
|
||||||
// in which we split from storing just the data to storing both
|
|
||||||
// the data and metadata in two files.
|
|
||||||
versionWithDataAndMetaFiles = 1
|
|
||||||
// versionWithNameInMeta points to the backup format version where we begin
|
|
||||||
// storing files in kopia with their item ID instead of their OneDrive file
|
|
||||||
// name.
|
|
||||||
// TODO(ashmrtn): Update this to a real value when we merge the file name
|
|
||||||
// change. Set to MAXINT for now to keep the if-check using it working.
|
|
||||||
versionWithNameInMeta = math.MaxInt
|
|
||||||
)
|
|
||||||
|
|
||||||
func getParentPermissions(
|
func getParentPermissions(
|
||||||
parentPath path.Path,
|
parentPath path.Path,
|
||||||
@ -288,7 +275,7 @@ func RestoreCollection(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if source == OneDriveSource && backupVersion >= versionWithDataAndMetaFiles {
|
if source == OneDriveSource && backupVersion >= version.OneDrive1DataAndMetaFiles {
|
||||||
name := itemData.UUID()
|
name := itemData.UUID()
|
||||||
|
|
||||||
if strings.HasSuffix(name, DataFileSuffix) {
|
if strings.HasSuffix(name, DataFileSuffix) {
|
||||||
@ -300,7 +287,7 @@ func RestoreCollection(
|
|||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
|
|
||||||
if backupVersion < versionWithNameInMeta {
|
if backupVersion < version.OneDriveXNameInMeta {
|
||||||
itemInfo, err = restoreV1File(
|
itemInfo, err = restoreV1File(
|
||||||
ctx,
|
ctx,
|
||||||
source,
|
source,
|
||||||
|
|||||||
@ -27,6 +27,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/kopia"
|
"github.com/alcionai/corso/src/internal/kopia"
|
||||||
"github.com/alcionai/corso/src/internal/model"
|
"github.com/alcionai/corso/src/internal/model"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
"github.com/alcionai/corso/src/pkg/backup"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
@ -341,7 +342,7 @@ func generateContainerOfItems(
|
|||||||
|
|
||||||
deets, err := gc.RestoreDataCollections(
|
deets, err := gc.RestoreDataCollections(
|
||||||
ctx,
|
ctx,
|
||||||
backup.Version,
|
version.Backup,
|
||||||
acct,
|
acct,
|
||||||
sel,
|
sel,
|
||||||
dest,
|
dest,
|
||||||
|
|||||||
37
src/internal/version/backup.go
Normal file
37
src/internal/version/backup.go
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
package version
|
||||||
|
|
||||||
|
import "math"
|
||||||
|
|
||||||
|
const Backup = 3
|
||||||
|
|
||||||
|
// Various labels to refer to important version changes.
|
||||||
|
// Labels don't need 1:1 service:version representation. Add a new
|
||||||
|
// label when it's important to mark a delta in behavior that's handled
|
||||||
|
// somewhere in the logic.
|
||||||
|
// Labels should state their application, the backup version number,
|
||||||
|
// and the colloquial purpose of the label.
|
||||||
|
const (
|
||||||
|
// OneDrive1DataAndMetaFiles is the corso backup format version
|
||||||
|
// in which we split from storing just the data to storing both
|
||||||
|
// the data and metadata in two files.
|
||||||
|
OneDrive1DataAndMetaFiles = 1
|
||||||
|
|
||||||
|
// OneDrive3IsMetaMarker is a small improvement on
|
||||||
|
// VersionWithDataAndMetaFiles, but has a marker IsMeta which
|
||||||
|
// specifies if the file is a meta file or a data file.
|
||||||
|
OneDrive3IsMetaMarker = 3
|
||||||
|
|
||||||
|
// OneDrive4IncludesPermissions includes permissions in the backup.
|
||||||
|
// Note that this is larger than the current backup version. That's
|
||||||
|
// because it isn't implemented yet. But we have tests based on this,
|
||||||
|
// so maybe we just keep bumping the verson ahead of the backup until
|
||||||
|
// it gets implemented.
|
||||||
|
OneDriveXIncludesPermissions = Backup + 1
|
||||||
|
|
||||||
|
// OneDriveXNameInMeta points to the backup format version where we begin
|
||||||
|
// storing files in kopia with their item ID instead of their OneDrive file
|
||||||
|
// name.
|
||||||
|
// TODO(ashmrtn): Update this to a real value when we merge the file name
|
||||||
|
// change. Set to MAXINT for now to keep the if-check using it working.
|
||||||
|
OneDriveXNameInMeta = math.MaxInt
|
||||||
|
)
|
||||||
@ -10,12 +10,11 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
"github.com/alcionai/corso/src/internal/model"
|
"github.com/alcionai/corso/src/internal/model"
|
||||||
"github.com/alcionai/corso/src/internal/stats"
|
"github.com/alcionai/corso/src/internal/stats"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
)
|
)
|
||||||
|
|
||||||
const Version = 2
|
|
||||||
|
|
||||||
// Backup represents the result of a backup operation
|
// Backup represents the result of a backup operation
|
||||||
type Backup struct {
|
type Backup struct {
|
||||||
model.BaseModel
|
model.BaseModel
|
||||||
@ -72,7 +71,7 @@ func New(
|
|||||||
Errors: errs.Data(),
|
Errors: errs.Data(),
|
||||||
ReadWrites: rw,
|
ReadWrites: rw,
|
||||||
StartAndEndTime: se,
|
StartAndEndTime: se,
|
||||||
Version: Version,
|
Version: version.Backup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -35,10 +35,12 @@ type DetailsModel struct {
|
|||||||
// Print writes the DetailModel Entries to StdOut, in the format
|
// Print writes the DetailModel Entries to StdOut, in the format
|
||||||
// requested by the caller.
|
// requested by the caller.
|
||||||
func (dm DetailsModel) PrintEntries(ctx context.Context) {
|
func (dm DetailsModel) PrintEntries(ctx context.Context) {
|
||||||
|
sl := dm.FilterMetaFiles()
|
||||||
|
|
||||||
if print.JSONFormat() {
|
if print.JSONFormat() {
|
||||||
printJSON(ctx, dm)
|
printJSON(ctx, sl)
|
||||||
} else {
|
} else {
|
||||||
printTable(ctx, dm)
|
printTable(ctx, sl)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,13 +73,13 @@ func printJSON(ctx context.Context, dm DetailsModel) {
|
|||||||
print.All(ctx, ents...)
|
print.All(ctx, ents...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Paths returns the list of Paths for non-folder items extracted from the
|
// Paths returns the list of Paths for non-folder and non-meta items extracted
|
||||||
// Entries slice.
|
// from the Entries slice.
|
||||||
func (dm DetailsModel) Paths() []string {
|
func (dm DetailsModel) Paths() []string {
|
||||||
r := make([]string, 0, len(dm.Entries))
|
r := make([]string, 0, len(dm.Entries))
|
||||||
|
|
||||||
for _, ent := range dm.Entries {
|
for _, ent := range dm.Entries {
|
||||||
if ent.Folder != nil {
|
if ent.Folder != nil || ent.isMetaFile() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,21 +91,49 @@ func (dm DetailsModel) Paths() []string {
|
|||||||
|
|
||||||
// Items returns a slice of *ItemInfo that does not contain any FolderInfo
|
// Items returns a slice of *ItemInfo that does not contain any FolderInfo
|
||||||
// entries. Required because not all folders in the details are valid resource
|
// entries. Required because not all folders in the details are valid resource
|
||||||
// paths.
|
// paths, and we want to slice out metadata.
|
||||||
func (dm DetailsModel) Items() []*DetailsEntry {
|
func (dm DetailsModel) Items() []*DetailsEntry {
|
||||||
res := make([]*DetailsEntry, 0, len(dm.Entries))
|
res := make([]*DetailsEntry, 0, len(dm.Entries))
|
||||||
|
|
||||||
for i := 0; i < len(dm.Entries); i++ {
|
for i := 0; i < len(dm.Entries); i++ {
|
||||||
if dm.Entries[i].Folder != nil {
|
ent := dm.Entries[i]
|
||||||
|
if ent.Folder != nil || ent.isMetaFile() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
res = append(res, &dm.Entries[i])
|
res = append(res, &ent)
|
||||||
}
|
}
|
||||||
|
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FilterMetaFiles returns a copy of the Details with all of the
|
||||||
|
// .meta files removed from the entries.
|
||||||
|
func (dm DetailsModel) FilterMetaFiles() DetailsModel {
|
||||||
|
d2 := DetailsModel{
|
||||||
|
Entries: []DetailsEntry{},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ent := range dm.Entries {
|
||||||
|
if !ent.isMetaFile() {
|
||||||
|
d2.Entries = append(d2.Entries, ent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return d2
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if a file is a metadata file. These are used to store
|
||||||
|
// additional data like permissions in case of OneDrive and are not to
|
||||||
|
// be treated as regular files.
|
||||||
|
func (de DetailsEntry) isMetaFile() bool {
|
||||||
|
return de.ItemInfo.OneDrive != nil && de.ItemInfo.OneDrive.IsMeta
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Builder
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
// Builder should be used to create a details model.
|
// Builder should be used to create a details model.
|
||||||
type Builder struct {
|
type Builder struct {
|
||||||
d Details
|
d Details
|
||||||
@ -583,6 +613,7 @@ type OneDriveInfo struct {
|
|||||||
ItemName string `json:"itemName,omitempty"`
|
ItemName string `json:"itemName,omitempty"`
|
||||||
DriveName string `json:"driveName,omitempty"`
|
DriveName string `json:"driveName,omitempty"`
|
||||||
ItemType ItemType `json:"itemType,omitempty"`
|
ItemType ItemType `json:"itemType,omitempty"`
|
||||||
|
IsMeta bool `json:"isMeta,omitempty"`
|
||||||
Modified time.Time `json:"modified,omitempty"`
|
Modified time.Time `json:"modified,omitempty"`
|
||||||
Owner string `json:"owner,omitempty"`
|
Owner string `json:"owner,omitempty"`
|
||||||
ParentPath string `json:"parentPath"`
|
ParentPath string `json:"parentPath"`
|
||||||
|
|||||||
@ -224,6 +224,69 @@ var pathItemsTable = []struct {
|
|||||||
expectRepoRefs: []string{"abcde", "12345"},
|
expectRepoRefs: []string{"abcde", "12345"},
|
||||||
expectLocationRefs: []string{"locationref", "locationref2"},
|
expectLocationRefs: []string{"locationref", "locationref2"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "multiple entries with meta file",
|
||||||
|
ents: []DetailsEntry{
|
||||||
|
{
|
||||||
|
RepoRef: "abcde",
|
||||||
|
LocationRef: "locationref",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "foo.meta",
|
||||||
|
LocationRef: "locationref.dirmeta",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "is-meta-file",
|
||||||
|
LocationRef: "locationref-meta-file",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectRepoRefs: []string{"abcde", "foo.meta"},
|
||||||
|
expectLocationRefs: []string{"locationref", "locationref.dirmeta"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple entries with folder and meta file",
|
||||||
|
ents: []DetailsEntry{
|
||||||
|
{
|
||||||
|
RepoRef: "abcde",
|
||||||
|
LocationRef: "locationref",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "12345",
|
||||||
|
LocationRef: "locationref2",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "foo.meta",
|
||||||
|
LocationRef: "locationref.dirmeta",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "is-meta-file",
|
||||||
|
LocationRef: "locationref-meta-file",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "deadbeef",
|
||||||
|
LocationRef: "locationref3",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
Folder: &FolderInfo{
|
||||||
|
DisplayName: "test folder",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectRepoRefs: []string{"abcde", "12345", "foo.meta"},
|
||||||
|
expectLocationRefs: []string{"locationref", "locationref2", "locationref.dirmeta"},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
|
func (suite *DetailsUnitSuite) TestDetailsModel_Path() {
|
||||||
@ -259,6 +322,38 @@ func (suite *DetailsUnitSuite) TestDetailsModel_Items() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *DetailsUnitSuite) TestDetailsModel_FilterMetaFiles() {
|
||||||
|
t := suite.T()
|
||||||
|
|
||||||
|
d := &DetailsModel{
|
||||||
|
Entries: []DetailsEntry{
|
||||||
|
{
|
||||||
|
RepoRef: "a.data",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "b.meta",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
RepoRef: "c.meta",
|
||||||
|
ItemInfo: ItemInfo{
|
||||||
|
OneDrive: &OneDriveInfo{IsMeta: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
d2 := d.FilterMetaFiles()
|
||||||
|
|
||||||
|
assert.Len(t, d2.Entries, 2)
|
||||||
|
assert.Len(t, d.Entries, 3)
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
|
func (suite *DetailsUnitSuite) TestDetails_AddFolders() {
|
||||||
itemTime := time.Date(2022, 10, 21, 10, 0, 0, 0, time.UTC)
|
itemTime := time.Date(2022, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||||
folderTimeOlderThanItem := time.Date(2022, 9, 21, 10, 0, 0, 0, time.UTC)
|
folderTimeOlderThanItem := time.Date(2022, 9, 21, 10, 0, 0, 0, time.UTC)
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package repository
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
@ -9,12 +10,14 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common/crash"
|
"github.com/alcionai/corso/src/internal/common/crash"
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||||
"github.com/alcionai/corso/src/internal/events"
|
"github.com/alcionai/corso/src/internal/events"
|
||||||
"github.com/alcionai/corso/src/internal/kopia"
|
"github.com/alcionai/corso/src/internal/kopia"
|
||||||
"github.com/alcionai/corso/src/internal/model"
|
"github.com/alcionai/corso/src/internal/model"
|
||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
"github.com/alcionai/corso/src/internal/operations"
|
"github.com/alcionai/corso/src/internal/operations"
|
||||||
"github.com/alcionai/corso/src/internal/streamstore"
|
"github.com/alcionai/corso/src/internal/streamstore"
|
||||||
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
"github.com/alcionai/corso/src/pkg/backup"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
@ -358,6 +361,20 @@ func (r repository) BackupDetails(
|
|||||||
return nil, nil, errs.Fail(err)
|
return nil, nil, errs.Fail(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retroactively fill in isMeta information for items in older
|
||||||
|
// backup versions without that info
|
||||||
|
// version.Restore2 introduces the IsMeta flag, so only v1 needs a check.
|
||||||
|
if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker {
|
||||||
|
for _, d := range deets.Entries {
|
||||||
|
if d.OneDrive != nil {
|
||||||
|
if strings.HasSuffix(d.RepoRef, onedrive.MetaFileSuffix) ||
|
||||||
|
strings.HasSuffix(d.RepoRef, onedrive.DirMetaFileSuffix) {
|
||||||
|
d.OneDrive.IsMeta = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return deets, b, errs
|
return deets, b, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user