Create abstraction to represent a unique location (#3100)

UniqueLocation allows representing a folder path that has both a location that will be stored in backup details and a location that can be used as a key in maps. The key for maps is guaranteed to be unique for all data types within a service Add a function to extract a unique location from a backup details entry and tests for that UniqueLocation will eventually be used in GraphConnector and KopiaWrapper so it needs to be in a package that both of them can import --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change - [x] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Issue(s) * #2486 #### Test Plan - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
2023-04-12 11:08:46 -07:00 · 2023-04-12 11:08:46 -07:00 · 9e692c7e2e
commit 9e692c7e2e
parent da8ac5cdbc
8 changed files with 385 additions and 15 deletions
--- a/src/internal/operations/manifests_test.go
+++ b/src/internal/operations/manifests_test.go
@ -933,9 +933,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
 					incomplete = "ir"
 				}

-				mn := makeMan(m.id, incomplete, mainReasons)
-				t.Logf("adding manifest (%p)\n%v\n%v\n\n", mn, *mn.Manifest, mn.Reasons)
-
 				mans = append(mans, makeMan(m.id, incomplete, mainReasons))
 			}

@ -945,9 +942,6 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
 					incomplete = "ir"
 				}

-				mn := makeMan(m.id, incomplete, fbReasons)
-				t.Logf("adding manifest (%p)\n%v\n%v\n\n", mn, *mn.Manifest, mn.Reasons)
-
 				mans = append(mans, makeMan(m.id, incomplete, fbReasons))
 			}

--- a/src/internal/version/backup.go
+++ b/src/internal/version/backup.go
@ -14,6 +14,9 @@ const (
 	// the data and metadata in two files.
 	OneDrive1DataAndMetaFiles = 1

+	// Version 2 switched Exchange calendars from using folder display names to
+	// folder IDs in their RepoRef.
+
 	// OneDrive3IsMetaMarker is a small improvement on
 	// VersionWithDataAndMetaFiles, but has a marker IsMeta which
 	// specifies if the file is a meta file or a data file.
@ -32,4 +35,8 @@ const (
 	// storing files in kopia with their item ID instead of their OneDrive file
 	// name.
 	OneDrive6NameInMeta = 6
+
+	// OneDriveXLocationRef provides LocationRef information for Exchange,
+	// OneDrive, and SharePoint libraries.
+	OneDriveXLocationRef = Backup + 1
 )
--- a/src/pkg/backup/details/details.go
+++ b/src/pkg/backup/details/details.go
@ -13,9 +13,88 @@ import (

 	"github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/internal/common"
+	"github.com/alcionai/corso/src/internal/version"
 	"github.com/alcionai/corso/src/pkg/path"
 )

+// LocationIDer provides access to location information but guarantees that it
+// can also generate a unique location (among items in the same service but
+// possibly across data types within the service) that can be used as a key in
+// maps and other structures. The unique location may be different than
+// InDetails, the location used in backup details.
+type LocationIDer interface {
+	ID() *path.Builder
+	InDetails() *path.Builder
+}
+
+type uniqueLoc struct {
+	pb          *path.Builder
+	prefixElems int
+}
+
+func (ul uniqueLoc) ID() *path.Builder {
+	return ul.pb
+}
+
+func (ul uniqueLoc) InDetails() *path.Builder {
+	return path.Builder{}.Append(ul.pb.Elements()[ul.prefixElems:]...)
+}
+
+// Having service-specific constructors can be kind of clunky, but in this case
+// I think they'd be useful to ensure the proper args are used since this
+// path.Builder is used as a key in some maps.
+
+// NewExchangeLocationIDer builds a LocationIDer for the given category and
+// folder path. The path denoted by the folders should be unique within the
+// category.
+func NewExchangeLocationIDer(
+	category path.CategoryType,
+	escapedFolders ...string,
+) (uniqueLoc, error) {
+	if err := path.ValidateServiceAndCategory(path.ExchangeService, category); err != nil {
+		return uniqueLoc{}, clues.Wrap(err, "making exchange LocationIDer")
+	}
+
+	pb := path.Builder{}.Append(category.String()).Append(escapedFolders...)
+
+	return uniqueLoc{
+		pb:          pb,
+		prefixElems: 1,
+	}, nil
+}
+
+// NewOneDriveLocationIDer builds a LocationIDer for the drive and folder path.
+// The path denoted by the folders should be unique within the drive.
+func NewOneDriveLocationIDer(
+	driveID string,
+	escapedFolders ...string,
+) uniqueLoc {
+	pb := path.Builder{}.
+		Append(path.FilesCategory.String(), driveID).
+		Append(escapedFolders...)
+
+	return uniqueLoc{
+		pb:          pb,
+		prefixElems: 2,
+	}
+}
+
+// NewSharePointLocationIDer builds a LocationIDer for the drive and folder
+// path. The path denoted by the folders should be unique within the drive.
+func NewSharePointLocationIDer(
+	driveID string,
+	escapedFolders ...string,
+) uniqueLoc {
+	pb := path.Builder{}.
+		Append(path.LibrariesCategory.String(), driveID).
+		Append(escapedFolders...)
+
+	return uniqueLoc{
+		pb:          pb,
+		prefixElems: 2,
+	}
+}
+
 type folderEntry struct {
 	RepoRef     string
 	ShortRef    string
@ -363,6 +442,52 @@ type DetailsEntry struct {
 	ItemInfo
 }

+// ToLocationIDer takes a backup version and produces the unique location for
+// this entry if possible. Reasons it may not be possible to produce the unique
+// location include an unsupported backup version or missing information.
+func (de DetailsEntry) ToLocationIDer(backupVersion int) (LocationIDer, error) {
+	if len(de.LocationRef) > 0 {
+		baseLoc, err := path.Builder{}.SplitUnescapeAppend(de.LocationRef)
+		if err != nil {
+			return nil, clues.Wrap(err, "parsing base location info").
+				With("location_ref", de.LocationRef)
+		}
+
+		// Individual services may add additional info to the base and return that.
+		return de.ItemInfo.uniqueLocation(baseLoc)
+	}
+
+	if backupVersion >= version.OneDriveXLocationRef ||
+		(de.ItemInfo.infoType() != OneDriveItem &&
+			de.ItemInfo.infoType() != SharePointLibrary) {
+		return nil, clues.New("no previous location for entry")
+	}
+
+	// This is a little hacky, but we only want to try to extract the old
+	// location if it's OneDrive or SharePoint libraries and it's known to
+	// be an older backup version.
+	//
+	// TODO(ashmrtn): Remove this code once OneDrive/SharePoint libraries
+	// LocationRef code has been out long enough that all delta tokens for
+	// previous backup versions will have expired. At that point, either
+	// we'll do a full backup (token expired, no newer backups) or have a
+	// backup of a higher version with the information we need.
+	rr, err := path.FromDataLayerPath(de.RepoRef, true)
+	if err != nil {
+		return nil, clues.Wrap(err, "getting item RepoRef")
+	}
+
+	p, err := path.ToOneDrivePath(rr)
+	if err != nil {
+		return nil, clues.New("converting RepoRef to OneDrive path")
+	}
+
+	baseLoc := path.Builder{}.Append(p.Root).Append(p.Folders...)
+
+	// Individual services may add additional info to the base and return that.
+	return de.ItemInfo.uniqueLocation(baseLoc)
+}
+
 // --------------------------------------------------------------------------------
 // CLI Output
 // --------------------------------------------------------------------------------
@ -541,6 +666,22 @@ func (i ItemInfo) Modified() time.Time {
 	return time.Time{}
 }

+func (i ItemInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) {
+	switch {
+	case i.Exchange != nil:
+		return i.Exchange.uniqueLocation(baseLoc)
+
+	case i.OneDrive != nil:
+		return i.OneDrive.uniqueLocation(baseLoc)
+
+	case i.SharePoint != nil:
+		return i.SharePoint.uniqueLocation(baseLoc)
+
+	default:
+		return nil, clues.New("unsupported type")
+	}
+}
+
 type FolderInfo struct {
 	ItemType    ItemType  `json:"itemType,omitempty"`
 	DisplayName string    `json:"displayName"`
@ -628,6 +769,21 @@ func (i *ExchangeInfo) UpdateParentPath(_ path.Path, locPath *path.Builder) erro
 	return nil
 }

+func (i *ExchangeInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) {
+	var category path.CategoryType
+
+	switch i.ItemType {
+	case ExchangeEvent:
+		category = path.EventsCategory
+	case ExchangeContact:
+		category = path.ContactsCategory
+	case ExchangeMail:
+		category = path.EmailCategory
+	}
+
+	return NewExchangeLocationIDer(category, baseLoc.Elements()...)
+}
+
 // SharePointInfo describes a sharepoint item
 type SharePointInfo struct {
 	Created    time.Time `json:"created,omitempty"`
@ -673,6 +829,14 @@ func (i *SharePointInfo) UpdateParentPath(newPath path.Path, _ *path.Builder) er
 	return nil
 }

+func (i *SharePointInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) {
+	if len(i.DriveID) == 0 {
+		return nil, clues.New("empty drive ID")
+	}
+
+	return NewSharePointLocationIDer(i.DriveID, baseLoc.Elements()...), nil
+}
+
 // OneDriveInfo describes a oneDrive item
 type OneDriveInfo struct {
 	Created    time.Time `json:"created,omitempty"`
@ -716,3 +880,11 @@ func (i *OneDriveInfo) UpdateParentPath(newPath path.Path, _ *path.Builder) erro

 	return nil
 }
+
+func (i *OneDriveInfo) uniqueLocation(baseLoc *path.Builder) (LocationIDer, error) {
+	if len(i.DriveID) == 0 {
+		return nil, clues.New("empty drive ID")
+	}
+
+	return NewOneDriveLocationIDer(i.DriveID, baseLoc.Elements()...), nil
+}
--- a/src/pkg/backup/details/details_test.go
+++ b/src/pkg/backup/details/details_test.go
@ -2,6 +2,7 @@ package details

 import (
 	"bytes"
+	"fmt"
 	"io"
 	"testing"
 	"time"
@ -13,6 +14,7 @@ import (

 	"github.com/alcionai/corso/src/internal/common"
 	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/version"
 	"github.com/alcionai/corso/src/pkg/path"
 )

@ -1222,3 +1224,195 @@ func (suite *DetailsUnitSuite) TestUnarshalTo() {
 		})
 	}
 }
+
+func (suite *DetailsUnitSuite) TestLocationIDer_FromEntry() {
+	const (
+		rrString = "tenant-id/%s/user-id/%s/drives/drive-id/root:/some/folder/stuff/item"
+		driveID  = "driveID"
+
+		expectedUniqueLocFmt         = "%s/" + driveID + "/root:/some/folder/stuff"
+		expectedExchangeUniqueLocFmt = "%s/root:/some/folder/stuff"
+		expectedDetailsLoc           = "root:/some/folder/stuff"
+	)
+
+	table := []struct {
+		name              string
+		service           string
+		category          string
+		itemInfo          ItemInfo
+		hasLocRef         bool
+		backupVersion     int
+		expectedErr       require.ErrorAssertionFunc
+		expectedUniqueLoc string
+	}{
+		{
+			name:     "OneDrive With Drive ID Old Version",
+			service:  path.OneDriveService.String(),
+			category: path.FilesCategory.String(),
+			itemInfo: ItemInfo{
+				OneDrive: &OneDriveInfo{
+					ItemType: OneDriveItem,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef - 1,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.FilesCategory),
+		},
+		{
+			name:     "OneDrive With Drive ID And LocationRef",
+			service:  path.OneDriveService.String(),
+			category: path.FilesCategory.String(),
+			itemInfo: ItemInfo{
+				OneDrive: &OneDriveInfo{
+					ItemType: OneDriveItem,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef,
+			hasLocRef:         true,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.FilesCategory),
+		},
+		{
+			name:     "OneDrive With Drive ID New Version Errors",
+			service:  path.OneDriveService.String(),
+			category: path.FilesCategory.String(),
+			itemInfo: ItemInfo{
+				OneDrive: &OneDriveInfo{
+					ItemType: OneDriveItem,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion: version.OneDriveXLocationRef,
+			expectedErr:   require.Error,
+		},
+		{
+			name:     "SharePoint With Drive ID Old Version",
+			service:  path.SharePointService.String(),
+			category: path.LibrariesCategory.String(),
+			itemInfo: ItemInfo{
+				SharePoint: &SharePointInfo{
+					ItemType: SharePointLibrary,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef - 1,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.LibrariesCategory),
+		},
+		{
+			name:     "SharePoint With Drive ID And LocationRef",
+			service:  path.SharePointService.String(),
+			category: path.LibrariesCategory.String(),
+			itemInfo: ItemInfo{
+				SharePoint: &SharePointInfo{
+					ItemType: SharePointLibrary,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef,
+			hasLocRef:         true,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedUniqueLocFmt, path.LibrariesCategory),
+		},
+		{
+			name:     "SharePoint With Drive ID New Version Errors",
+			service:  path.SharePointService.String(),
+			category: path.LibrariesCategory.String(),
+			itemInfo: ItemInfo{
+				SharePoint: &SharePointInfo{
+					ItemType: SharePointLibrary,
+					DriveID:  driveID,
+				},
+			},
+			backupVersion: version.OneDriveXLocationRef,
+			expectedErr:   require.Error,
+		},
+		{
+			name:     "Exchange Email With LocationRef Old Version",
+			service:  path.ExchangeService.String(),
+			category: path.EmailCategory.String(),
+			itemInfo: ItemInfo{
+				Exchange: &ExchangeInfo{
+					ItemType: ExchangeMail,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef - 1,
+			hasLocRef:         true,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedExchangeUniqueLocFmt, path.EmailCategory),
+		},
+		{
+			name:     "Exchange Email With LocationRef New Version",
+			service:  path.ExchangeService.String(),
+			category: path.EmailCategory.String(),
+			itemInfo: ItemInfo{
+				Exchange: &ExchangeInfo{
+					ItemType: ExchangeMail,
+				},
+			},
+			backupVersion:     version.OneDriveXLocationRef,
+			hasLocRef:         true,
+			expectedErr:       require.NoError,
+			expectedUniqueLoc: fmt.Sprintf(expectedExchangeUniqueLocFmt, path.EmailCategory),
+		},
+		{
+			name:     "Exchange Email Without LocationRef Old Version Errors",
+			service:  path.ExchangeService.String(),
+			category: path.EmailCategory.String(),
+			itemInfo: ItemInfo{
+				Exchange: &ExchangeInfo{
+					ItemType: ExchangeMail,
+				},
+			},
+			backupVersion: version.OneDriveXLocationRef - 1,
+			expectedErr:   require.Error,
+		},
+		{
+			name:     "Exchange Email Without LocationRef New Version Errors",
+			service:  path.ExchangeService.String(),
+			category: path.EmailCategory.String(),
+			itemInfo: ItemInfo{
+				Exchange: &ExchangeInfo{
+					ItemType: ExchangeMail,
+				},
+			},
+			backupVersion: version.OneDriveXLocationRef,
+			expectedErr:   require.Error,
+		},
+	}
+
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			entry := DetailsEntry{
+				RepoRef:  fmt.Sprintf(rrString, test.service, test.category),
+				ItemInfo: test.itemInfo,
+			}
+
+			if test.hasLocRef {
+				entry.LocationRef = expectedDetailsLoc
+			}
+
+			loc, err := entry.ToLocationIDer(test.backupVersion)
+			test.expectedErr(t, err, clues.ToCore(err))
+
+			if err != nil {
+				return
+			}
+
+			assert.Equal(
+				t,
+				test.expectedUniqueLoc,
+				loc.ID().String(),
+				"unique location")
+			assert.Equal(
+				t,
+				expectedDetailsLoc,
+				loc.InDetails().String(),
+				"details location")
+		})
+	}
+}
--- a/src/pkg/path/onedrive.go
+++ b/src/pkg/path/onedrive.go
@ -10,6 +10,7 @@ import "github.com/alcionai/clues"
 // folders[] is []{"Folder1", "Folder2"}
 type DrivePath struct {
 	DriveID string
+	Root    string
 	Folders Elements
 }

@ -23,7 +24,7 @@ func ToOneDrivePath(p Path) (*DrivePath, error) {
 			With("path_folders", p.Folder(false))
 	}

-	return &DrivePath{DriveID: folders[1], Folders: folders[3:]}, nil
+	return &DrivePath{DriveID: folders[1], Root: folders[2], Folders: folders[3:]}, nil
 }

 // Returns the path to the folder within the drive (i.e. under `root:`)
--- a/src/pkg/path/onedrive_test.go
+++ b/src/pkg/path/onedrive_test.go
@ -21,6 +21,8 @@ func TestOneDrivePathSuite(t *testing.T) {
 }

 func (suite *OneDrivePathSuite) Test_ToOneDrivePath() {
+	const root = "root:"
+
 	tests := []struct {
 		name         string
 		pathElements []string
@ -34,14 +36,14 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() {
 		},
 		{
 			name:         "Root path",
-			pathElements: []string{"drive", "driveID", "root:"},
-			expected:     &path.DrivePath{DriveID: "driveID", Folders: []string{}},
+			pathElements: []string{"drive", "driveID", root},
+			expected:     &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{}},
 			errCheck:     assert.NoError,
 		},
 		{
 			name:         "Deeper path",
-			pathElements: []string{"drive", "driveID", "root:", "folder1", "folder2"},
-			expected:     &path.DrivePath{DriveID: "driveID", Folders: []string{"folder1", "folder2"}},
+			pathElements: []string{"drive", "driveID", root, "folder1", "folder2"},
+			expected:     &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{"folder1", "folder2"}},
 			errCheck:     assert.NoError,
 		},
 	}
--- a/src/pkg/path/path.go
+++ b/src/pkg/path/path.go
@ -341,7 +341,7 @@ func (pb Builder) ToServiceCategoryMetadataPath(
 	category CategoryType,
 	isItem bool,
 ) (Path, error) {
-	if err := validateServiceAndCategory(service, category); err != nil {
+	if err := ValidateServiceAndCategory(service, category); err != nil {
 		return nil, err
 	}

@ -398,7 +398,7 @@ func (pb Builder) ToDataLayerPath(
 	category CategoryType,
 	isItem bool,
 ) (Path, error) {
-	if err := validateServiceAndCategory(service, category); err != nil {
+	if err := ValidateServiceAndCategory(service, category); err != nil {
 		return nil, err
 	}

--- a/src/pkg/path/resource_path.go
+++ b/src/pkg/path/resource_path.go
@ -128,14 +128,14 @@ func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType,
 		return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("category", fmt.Sprintf("%q", c))
 	}

-	if err := validateServiceAndCategory(service, category); err != nil {
+	if err := ValidateServiceAndCategory(service, category); err != nil {
 		return UnknownService, UnknownCategory, err
 	}

 	return service, category, nil
 }

-func validateServiceAndCategory(service ServiceType, category CategoryType) error {
+func ValidateServiceAndCategory(service ServiceType, category CategoryType) error {
 	cats, ok := serviceCategories[service]
 	if !ok {
 		return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))