From 9e692c7e2ed2b914c7746b9c7cec4dfab88e5264 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 12 Apr 2023 11:08:46 -0700 Subject: [PATCH] Create abstraction to represent a unique location (#3100) UniqueLocation allows representing a folder path that has both a location that will be stored in backup details and a location that can be used as a key in maps. The key for maps is guaranteed to be unique for all data types within a service Add a function to extract a unique location from a backup details entry and tests for that UniqueLocation will eventually be used in GraphConnector and KopiaWrapper so it needs to be in a package that both of them can import --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #2486 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/operations/manifests_test.go | 6 - src/internal/version/backup.go | 7 + src/pkg/backup/details/details.go | 172 +++++++++++++++++++ src/pkg/backup/details/details_test.go | 194 ++++++++++++++++++++++ src/pkg/path/onedrive.go | 3 +- src/pkg/path/onedrive_test.go | 10 +- src/pkg/path/path.go | 4 +- src/pkg/path/resource_path.go | 4 +- 8 files changed, 385 insertions(+), 15 deletions(-) diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index 0ca3aeb91..2e1b2d504 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -933,9 +933,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb incomplete = "ir" } - mn := makeMan(m.id, incomplete, mainReasons) - t.Logf("adding manifest (%p)\n%v\n%v\n\n", mn, *mn.Manifest, mn.Reasons) - mans = append(mans, makeMan(m.id, incomplete, mainReasons)) } @@ -945,9 +942,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb incomplete = "ir" } - mn := makeMan(m.id, incomplete, fbReasons) - t.Logf("adding manifest (%p)\n%v\n%v\n\n", mn, *mn.Manifest, mn.Reasons) - mans = append(mans, makeMan(m.id, incomplete, fbReasons)) } diff --git a/src/internal/version/backup.go b/src/internal/version/backup.go index 0ea734aca..9ba2d3660 100644 --- a/src/internal/version/backup.go +++ b/src/internal/version/backup.go @@ -14,6 +14,9 @@ const ( // the data and metadata in two files. OneDrive1DataAndMetaFiles = 1 + // Version 2 switched Exchange calendars from using folder display names to + // folder IDs in their RepoRef. + // OneDrive3IsMetaMarker is a small improvement on // VersionWithDataAndMetaFiles, but has a marker IsMeta which // specifies if the file is a meta file or a data file. @@ -32,4 +35,8 @@ const ( // storing files in kopia with their item ID instead of their OneDrive file // name. OneDrive6NameInMeta = 6 + + // OneDriveXLocationRef provides LocationRef information for Exchange, + // OneDrive, and SharePoint libraries. + OneDriveXLocationRef = Backup + 1 ) diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index 7cd99a7bd..561310569 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -13,9 +13,88 @@ import ( "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" ) +// LocationIDer provides access to location information but guarantees that it +// can also generate a unique location (among items in the same service but +// possibly across data types within the service) that can be used as a key in +// maps and other structures. The unique location may be different than +// InDetails, the location used in backup details. +type LocationIDer interface { + ID() *path.Builder + InDetails() *path.Builder +} + +type uniqueLoc struct { + pb *path.Builder + prefixElems int +} + +func (ul uniqueLoc) ID() *path.Builder { + return ul.pb +} + +func (ul uniqueLoc) InDetails() *path.Builder { + return path.Builder{}.Append(ul.pb.Elements()[ul.prefixElems:]...) +} + +// Having service-specific constructors can be kind of clunky, but in this case +// I think they'd be useful to ensure the proper args are used since this +// path.Builder is used as a key in some maps. + +// NewExchangeLocationIDer builds a LocationIDer for the given category and +// folder path. The path denoted by the folders should be unique within the +// category. +func NewExchangeLocationIDer( + category path.CategoryType, + escapedFolders ...string, +) (uniqueLoc, error) { + if err := path.ValidateServiceAndCategory(path.ExchangeService, category); err != nil { + return uniqueLoc{}, clues.Wrap(err, "making exchange LocationIDer") + } + + pb := path.Builder{}.Append(category.String()).Append(escapedFolders...) + + return uniqueLoc{ + pb: pb, + prefixElems: 1, + }, nil +} + +// NewOneDriveLocationIDer builds a LocationIDer for the drive and folder path. +// The path denoted by the folders should be unique within the drive. +func NewOneDriveLocationIDer( + driveID string, + escapedFolders ...string, +) uniqueLoc { + pb := path.Builder{}. + Append(path.FilesCategory.String(), driveID). + Append(escapedFolders...) + + return uniqueLoc{ + pb: pb, + prefixElems: 2, + } +} + +// NewSharePointLocationIDer builds a LocationIDer for the drive and folder +// path. The path denoted by the folders should be unique within the drive. +func NewSharePointLocationIDer( + driveID string, + escapedFolders ...string, +) uniqueLoc { + pb := path.Builder{}. + Append(path.LibrariesCategory.String(), driveID). + Append(escapedFolders...) + + return uniqueLoc{ + pb: pb, + prefixElems: 2, + } +} + type folderEntry struct { RepoRef string ShortRef string @@ -363,6 +442,52 @@ type DetailsEntry struct { ItemInfo } +// ToLocationIDer takes a backup version and produces the unique location for +// this entry if possible. Reasons it may not be possible to produce the unique +// location include an unsupported backup version or missing information. +func (de DetailsEntry) ToLocationIDer(backupVersion int) (LocationIDer, error) { + if len(de.LocationRef) > 0 { + baseLoc, err := path.Builder{}.SplitUnescapeAppend(de.LocationRef) + if err != nil { + return nil, clues.Wrap(err, "parsing base location info"). + With("location_ref", de.LocationRef) + } + + // Individual services may add additional info to the base and return that. + return de.ItemInfo.uniqueLocation(baseLoc) + } + + if backupVersion >= version.OneDriveXLocationRef || + (de.ItemInfo.infoType() != OneDriveItem && + de.ItemInfo.infoType() != SharePointLibrary) { + return nil, clues.New("no previous location for entry") + } + + // This is a little hacky, but we only want to try to extract the old + // location if it's OneDrive or SharePoint libraries and it's known to + // be an older backup version. + // + // TODO(ashmrtn): Remove this code once OneDrive/SharePoint libraries + // LocationRef code has been out long enough that all delta tokens for + // previous backup versions will have expired. At that point, either + // we'll do a full backup (token expired, no newer backups) or have a + // backup of a higher version with the information we need. + rr, err := path.FromDataLayerPath(de.RepoRef, true) + if err != nil { + return nil, clues.Wrap(err, "getting item RepoRef") + } + + p, err := path.ToOneDrivePath(rr) + if err != nil { + return nil, clues.New("converting RepoRef to OneDrive path") + } + + baseLoc := path.Builder{}.Append(p.Root).Append(p.Folders...) + + // Individual services may add additional info to the base and return that. + return de.ItemInfo.uniqueLocation(baseLoc) +} + // -------------------------------------------------------------------------------- // CLI Output // -------------------------------------------------------------------------------- @@ -541,6 +666,22 @@ func (i ItemInfo) Modified() time.Time { return time.Time{} } +func (i ItemInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) { + switch { + case i.Exchange != nil: + return i.Exchange.uniqueLocation(baseLoc) + + case i.OneDrive != nil: + return i.OneDrive.uniqueLocation(baseLoc) + + case i.SharePoint != nil: + return i.SharePoint.uniqueLocation(baseLoc) + + default: + return nil, clues.New("unsupported type") + } +} + type FolderInfo struct { ItemType ItemType `json:"itemType,omitempty"` DisplayName string `json:"displayName"` @@ -628,6 +769,21 @@ func (i *ExchangeInfo) UpdateParentPath(_ path.Path, locPath *path.Builder) erro return nil } +func (i *ExchangeInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) { + var category path.CategoryType + + switch i.ItemType { + case ExchangeEvent: + category = path.EventsCategory + case ExchangeContact: + category = path.ContactsCategory + case ExchangeMail: + category = path.EmailCategory + } + + return NewExchangeLocationIDer(category, baseLoc.Elements()...) +} + // SharePointInfo describes a sharepoint item type SharePointInfo struct { Created time.Time `json:"created,omitempty"` @@ -673,6 +829,14 @@ func (i *SharePointInfo) UpdateParentPath(newPath path.Path, _ *path.Builder) er return nil } +func (i *SharePointInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) { + if len(i.DriveID) == 0 { + return nil, clues.New("empty drive ID") + } + + return NewSharePointLocationIDer(i.DriveID, baseLoc.Elements()...), nil +} + // OneDriveInfo describes a oneDrive item type OneDriveInfo struct { Created time.Time `json:"created,omitempty"` @@ -716,3 +880,11 @@ func (i *OneDriveInfo) UpdateParentPath(newPath path.Path, _ *path.Builder) erro return nil } + +func (i *OneDriveInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) { + if len(i.DriveID) == 0 { + return nil, clues.New("empty drive ID") + } + + return NewOneDriveLocationIDer(i.DriveID, baseLoc.Elements()...), nil +} diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index c3d1a494f..96c69c151 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -2,6 +2,7 @@ package details import ( "bytes" + "fmt" "io" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" ) @@ -1222,3 +1224,195 @@ func (suite *DetailsUnitSuite) TestUnarshalTo() { }) } } + +func (suite *DetailsUnitSuite) TestLocationIDer_FromEntry() { + const ( + rrString = "tenant-id/%s/user-id/%s/drives/drive-id/root:/some/folder/stuff/item" + driveID = "driveID" + + expectedUniqueLocFmt = "%s/" + driveID + "/root:/some/folder/stuff" + expectedExchangeUniqueLocFmt = "%s/root:/some/folder/stuff" + expectedDetailsLoc = "root:/some/folder/stuff" + ) + + table := []struct { + name string + service string + category string + itemInfo ItemInfo + hasLocRef bool + backupVersion int + expectedErr require.ErrorAssertionFunc + expectedUniqueLoc string + }{ + { + name: "OneDrive With Drive ID Old Version", + service: path.OneDriveService.String(), + category: path.FilesCategory.String(), + itemInfo: ItemInfo{ + OneDrive: &OneDriveInfo{ + ItemType: OneDriveItem, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef - 1, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.FilesCategory), + }, + { + name: "OneDrive With Drive ID And LocationRef", + service: path.OneDriveService.String(), + category: path.FilesCategory.String(), + itemInfo: ItemInfo{ + OneDrive: &OneDriveInfo{ + ItemType: OneDriveItem, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef, + hasLocRef: true, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.FilesCategory), + }, + { + name: "OneDrive With Drive ID New Version Errors", + service: path.OneDriveService.String(), + category: path.FilesCategory.String(), + itemInfo: ItemInfo{ + OneDrive: &OneDriveInfo{ + ItemType: OneDriveItem, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef, + expectedErr: require.Error, + }, + { + name: "SharePoint With Drive ID Old Version", + service: path.SharePointService.String(), + category: path.LibrariesCategory.String(), + itemInfo: ItemInfo{ + SharePoint: &SharePointInfo{ + ItemType: SharePointLibrary, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef - 1, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.LibrariesCategory), + }, + { + name: "SharePoint With Drive ID And LocationRef", + service: path.SharePointService.String(), + category: path.LibrariesCategory.String(), + itemInfo: ItemInfo{ + SharePoint: &SharePointInfo{ + ItemType: SharePointLibrary, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef, + hasLocRef: true, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.LibrariesCategory), + }, + { + name: "SharePoint With Drive ID New Version Errors", + service: path.SharePointService.String(), + category: path.LibrariesCategory.String(), + itemInfo: ItemInfo{ + SharePoint: &SharePointInfo{ + ItemType: SharePointLibrary, + DriveID: driveID, + }, + }, + backupVersion: version.OneDriveXLocationRef, + expectedErr: require.Error, + }, + { + name: "Exchange Email With LocationRef Old Version", + service: path.ExchangeService.String(), + category: path.EmailCategory.String(), + itemInfo: ItemInfo{ + Exchange: &ExchangeInfo{ + ItemType: ExchangeMail, + }, + }, + backupVersion: version.OneDriveXLocationRef - 1, + hasLocRef: true, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedExchangeUniqueLocFmt, path.EmailCategory), + }, + { + name: "Exchange Email With LocationRef New Version", + service: path.ExchangeService.String(), + category: path.EmailCategory.String(), + itemInfo: ItemInfo{ + Exchange: &ExchangeInfo{ + ItemType: ExchangeMail, + }, + }, + backupVersion: version.OneDriveXLocationRef, + hasLocRef: true, + expectedErr: require.NoError, + expectedUniqueLoc: fmt.Sprintf(expectedExchangeUniqueLocFmt, path.EmailCategory), + }, + { + name: "Exchange Email Without LocationRef Old Version Errors", + service: path.ExchangeService.String(), + category: path.EmailCategory.String(), + itemInfo: ItemInfo{ + Exchange: &ExchangeInfo{ + ItemType: ExchangeMail, + }, + }, + backupVersion: version.OneDriveXLocationRef - 1, + expectedErr: require.Error, + }, + { + name: "Exchange Email Without LocationRef New Version Errors", + service: path.ExchangeService.String(), + category: path.EmailCategory.String(), + itemInfo: ItemInfo{ + Exchange: &ExchangeInfo{ + ItemType: ExchangeMail, + }, + }, + backupVersion: version.OneDriveXLocationRef, + expectedErr: require.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + entry := DetailsEntry{ + RepoRef: fmt.Sprintf(rrString, test.service, test.category), + ItemInfo: test.itemInfo, + } + + if test.hasLocRef { + entry.LocationRef = expectedDetailsLoc + } + + loc, err := entry.ToLocationIDer(test.backupVersion) + test.expectedErr(t, err, clues.ToCore(err)) + + if err != nil { + return + } + + assert.Equal( + t, + test.expectedUniqueLoc, + loc.ID().String(), + "unique location") + assert.Equal( + t, + expectedDetailsLoc, + loc.InDetails().String(), + "details location") + }) + } +} diff --git a/src/pkg/path/onedrive.go b/src/pkg/path/onedrive.go index 76d5300be..48c443311 100644 --- a/src/pkg/path/onedrive.go +++ b/src/pkg/path/onedrive.go @@ -10,6 +10,7 @@ import "github.com/alcionai/clues" // folders[] is []{"Folder1", "Folder2"} type DrivePath struct { DriveID string + Root string Folders Elements } @@ -23,7 +24,7 @@ func ToOneDrivePath(p Path) (*DrivePath, error) { With("path_folders", p.Folder(false)) } - return &DrivePath{DriveID: folders[1], Folders: folders[3:]}, nil + return &DrivePath{DriveID: folders[1], Root: folders[2], Folders: folders[3:]}, nil } // Returns the path to the folder within the drive (i.e. under `root:`) diff --git a/src/pkg/path/onedrive_test.go b/src/pkg/path/onedrive_test.go index 9a44a0e7a..d81c59e31 100644 --- a/src/pkg/path/onedrive_test.go +++ b/src/pkg/path/onedrive_test.go @@ -21,6 +21,8 @@ func TestOneDrivePathSuite(t *testing.T) { } func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { + const root = "root:" + tests := []struct { name string pathElements []string @@ -34,14 +36,14 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }, { name: "Root path", - pathElements: []string{"drive", "driveID", "root:"}, - expected: &path.DrivePath{DriveID: "driveID", Folders: []string{}}, + pathElements: []string{"drive", "driveID", root}, + expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{}}, errCheck: assert.NoError, }, { name: "Deeper path", - pathElements: []string{"drive", "driveID", "root:", "folder1", "folder2"}, - expected: &path.DrivePath{DriveID: "driveID", Folders: []string{"folder1", "folder2"}}, + pathElements: []string{"drive", "driveID", root, "folder1", "folder2"}, + expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{"folder1", "folder2"}}, errCheck: assert.NoError, }, } diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 0a1decfa2..ee7a0303a 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -341,7 +341,7 @@ func (pb Builder) ToServiceCategoryMetadataPath( category CategoryType, isItem bool, ) (Path, error) { - if err := validateServiceAndCategory(service, category); err != nil { + if err := ValidateServiceAndCategory(service, category); err != nil { return nil, err } @@ -398,7 +398,7 @@ func (pb Builder) ToDataLayerPath( category CategoryType, isItem bool, ) (Path, error) { - if err := validateServiceAndCategory(service, category); err != nil { + if err := ValidateServiceAndCategory(service, category); err != nil { return nil, err } diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index a9db84d88..a352b34ab 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -128,14 +128,14 @@ func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType, return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("category", fmt.Sprintf("%q", c)) } - if err := validateServiceAndCategory(service, category); err != nil { + if err := ValidateServiceAndCategory(service, category); err != nil { return UnknownService, UnknownCategory, err } return service, category, nil } -func validateServiceAndCategory(service ServiceType, category CategoryType) error { +func ValidateServiceAndCategory(service ServiceType, category CategoryType) error { cats, ok := serviceCategories[service] if !ok { return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))