diff --git a/src/internal/common/keys/keys.go b/src/internal/common/keys/keys.go new file mode 100644 index 000000000..07d0cf444 --- /dev/null +++ b/src/internal/common/keys/keys.go @@ -0,0 +1,31 @@ +package keys + +type Set map[string]struct{} + +func (ks Set) HasKey(key string) bool { + if _, ok := ks[key]; ok { + return true + } + + return false +} + +func (ks Set) Keys() []string { + sliceKeys := make([]string, 0) + + for k := range ks { + sliceKeys = append(sliceKeys, k) + } + + return sliceKeys +} + +func HasKeys(data map[string]any, keys ...string) bool { + for _, k := range keys { + if _, ok := data[k]; !ok { + return false + } + } + + return true +} diff --git a/src/internal/common/keys/keys_test.go b/src/internal/common/keys/keys_test.go new file mode 100644 index 000000000..41e6de6d9 --- /dev/null +++ b/src/internal/common/keys/keys_test.go @@ -0,0 +1,122 @@ +package keys + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type KeySetTestSuite struct { + tester.Suite +} + +func TestKeySetTestSuite(t *testing.T) { + suite.Run(t, &KeySetTestSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *KeySetTestSuite) TestHasKey() { + tests := []struct { + name string + keySet Set + key string + expect assert.BoolAssertionFunc + }{ + { + name: "key exists in the set", + keySet: Set{"key1": {}, "key2": {}}, + key: "key1", + expect: assert.True, + }, + { + name: "key does not exist in the set", + keySet: Set{"key1": {}, "key2": {}}, + key: "nonexistent", + expect: assert.False, + }, + { + name: "empty set", + keySet: Set{}, + key: "key", + expect: assert.False, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + test.expect(suite.T(), test.keySet.HasKey(test.key)) + }) + } +} + +func (suite *KeySetTestSuite) TestKeys() { + tests := []struct { + name string + keySet Set + expect assert.ValueAssertionFunc + }{ + { + name: "non-empty set", + keySet: Set{"key1": {}, "key2": {}}, + expect: assert.NotEmpty, + }, + { + name: "empty set", + keySet: Set{}, + expect: assert.Empty, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + keys := test.keySet.Keys() + test.expect(suite.T(), keys, []string{"key1", "key2"}) + }) + } +} + +func (suite *KeySetTestSuite) TestHasKeys() { + tests := []struct { + name string + data map[string]any + keys []string + expect assert.BoolAssertionFunc + }{ + { + name: "has all keys", + data: map[string]any{ + "key1": "data1", + "key2": 2, + "key3": struct{}{}, + }, + keys: []string{"key1", "key2", "key3"}, + expect: assert.True, + }, + { + name: "has some keys", + data: map[string]any{ + "key1": "data1", + "key2": 2, + }, + keys: []string{"key1", "key2", "key3"}, + expect: assert.False, + }, + { + name: "has no key", + data: map[string]any{ + "key1": "data1", + "key2": 2, + }, + keys: []string{"key4", "key5", "key6"}, + expect: assert.False, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + test.expect(suite.T(), HasKeys(test.data, test.keys...)) + }) + } +} diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index 9e1d9d5c8..cca4d9ba3 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -134,7 +134,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { }, getItem: func(t *testing.T, itemName string) data.Item { byteArray := spMock.Page(itemName) - page, err := betaAPI.CreatePageFromBytes(byteArray) + page, err := betaAPI.BytesToSitePageable(byteArray) require.NoError(t, err, clues.ToCore(err)) data, err := data.NewPrefetchedItemWithInfo( diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index 2591c1340..5dd6c238e 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -154,7 +154,7 @@ func restoreListItem( return dii, clues.WrapWC(ctx, err, "reading backup data") } - oldList, err := betaAPI.CreateListFromBytes(byteArray) + oldList, err := api.BytesToListable(byteArray) if err != nil { return dii, clues.WrapWC(ctx, err, "creating item") } @@ -165,12 +165,12 @@ func restoreListItem( var ( newName = fmt.Sprintf("%s_%s", destName, listName) - newList = betaAPI.ToListable(oldList, newName) + newList = api.ToListable(oldList, newName) contents = make([]models.ListItemable, 0) ) for _, itm := range oldList.GetItems() { - temp := betaAPI.CloneListItem(itm) + temp := api.CloneListItem(itm) contents = append(contents, temp) } diff --git a/src/internal/m365/service/sharepoint/api/pages.go b/src/internal/m365/service/sharepoint/api/pages.go index 968ca6c56..32b8748a5 100644 --- a/src/internal/m365/service/sharepoint/api/pages.go +++ b/src/internal/m365/service/sharepoint/api/pages.go @@ -189,7 +189,7 @@ func RestoreSitePage( } // Hydrate Page - page, err := CreatePageFromBytes(byteArray) + page, err := BytesToSitePageable(byteArray) if err != nil { return dii, clues.WrapWC(ctx, err, "creating Page object") } @@ -257,3 +257,14 @@ func PageInfo(page betamodels.SitePageable, size int64) *details.SharePointInfo Size: size, } } + +func BytesToSitePageable(bytes []byte) (betamodels.SitePageable, error) { + parsable, err := createFromBytes(bytes, betamodels.CreateSitePageFromDiscriminatorValue) + if err != nil { + return nil, clues.Wrap(err, "deserializing bytes to sharepoint page") + } + + page := parsable.(betamodels.SitePageable) + + return page, nil +} diff --git a/src/internal/m365/service/sharepoint/api/pages_test.go b/src/internal/m365/service/sharepoint/api/pages_test.go index 0f9bb20af..0fdfbfb21 100644 --- a/src/internal/m365/service/sharepoint/api/pages_test.go +++ b/src/internal/m365/service/sharepoint/api/pages_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/alcionai/clues" + kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -21,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + bmodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService { @@ -34,6 +36,76 @@ func createTestBetaService(t *testing.T, credentials account.M365Config) *api.Be return api.NewBetaService(adapter) } +type SharepointPageUnitSuite struct { + tester.Suite +} + +func TestSharepointPageUnitSuite(t *testing.T) { + suite.Run(t, &SharepointPageUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharepointPageUnitSuite) TestCreatePageFromBytes() { + tests := []struct { + name string + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + getBytes func(t *testing.T) []byte + }{ + { + "empty bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return make([]byte, 0) + }, + }, + { + "invalid bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return []byte("snarf") + }, + }, + { + "Valid Page", + assert.NoError, + assert.NotNil, + func(t *testing.T) []byte { + pg := bmodels.NewSitePage() + title := "Tested" + pg.SetTitle(&title) + pg.SetName(&title) + pg.SetWebUrl(&title) + + writer := kioser.NewJsonSerializationWriter() + err := writer.WriteObjectValue("", pg) + require.NoError(t, err, clues.ToCore(err)) + + byteArray, err := writer.GetSerializedContent() + require.NoError(t, err, clues.ToCore(err)) + + return byteArray + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + result, err := api.BytesToSitePageable(test.getBytes(t)) + test.checkError(t, err) + test.isNil(t, result) + if result != nil { + assert.Equal(t, "Tested", *result.GetName(), "name") + assert.Equal(t, "Tested", *result.GetTitle(), "title") + assert.Equal(t, "Tested", *result.GetWebUrl(), "webURL") + } + }) + } +} + type SharePointPageSuite struct { tester.Suite siteID string diff --git a/src/internal/m365/service/sharepoint/api/serialization.go b/src/internal/m365/service/sharepoint/api/serialization.go index c125e0057..2410ca090 100644 --- a/src/internal/m365/service/sharepoint/api/serialization.go +++ b/src/internal/m365/service/sharepoint/api/serialization.go @@ -1,15 +1,9 @@ package api import ( - "strings" - "github.com/alcionai/clues" "github.com/microsoft/kiota-abstractions-go/serialization" kjson "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - betamodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) // createFromBytes generates an m365 object form bytes. @@ -29,184 +23,3 @@ func createFromBytes( return v, nil } - -func CreateListFromBytes(bytes []byte) (models.Listable, error) { - parsable, err := createFromBytes(bytes, models.CreateListFromDiscriminatorValue) - if err != nil { - return nil, clues.Wrap(err, "deserializing bytes to sharepoint list") - } - - list := parsable.(models.Listable) - - return list, nil -} - -func CreatePageFromBytes(bytes []byte) (betamodels.SitePageable, error) { - parsable, err := createFromBytes(bytes, betamodels.CreateSitePageFromDiscriminatorValue) - if err != nil { - return nil, clues.Wrap(err, "deserializing bytes to sharepoint page") - } - - page := parsable.(betamodels.SitePageable) - - return page, nil -} - -// ToListable utility function to encapsulate stored data for restoration. -// New Listable omits trackable fields such as `id` or `ETag` and other read-only -// objects that are prevented upon upload. Additionally, read-Only columns are -// not attached in this method. -// ListItems are not included in creation of new list, and have to be restored -// in separate call. -func ToListable(orig models.Listable, displayName string) models.Listable { - newList := models.NewList() - - newList.SetContentTypes(orig.GetContentTypes()) - newList.SetCreatedBy(orig.GetCreatedBy()) - newList.SetCreatedByUser(orig.GetCreatedByUser()) - newList.SetCreatedDateTime(orig.GetCreatedDateTime()) - newList.SetDescription(orig.GetDescription()) - newList.SetDisplayName(&displayName) - newList.SetLastModifiedBy(orig.GetLastModifiedBy()) - newList.SetLastModifiedByUser(orig.GetLastModifiedByUser()) - newList.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) - newList.SetList(orig.GetList()) - newList.SetOdataType(orig.GetOdataType()) - newList.SetParentReference(orig.GetParentReference()) - - columns := make([]models.ColumnDefinitionable, 0) - leg := map[string]struct{}{ - "Attachments": {}, - "Edit": {}, - "Content Type": {}, - } - - for _, cd := range orig.GetColumns() { - var ( - displayName string - readOnly bool - ) - - if name, ok := ptr.ValOK(cd.GetDisplayName()); ok { - displayName = name - } - - if ro, ok := ptr.ValOK(cd.GetReadOnly()); ok { - readOnly = ro - } - - _, isLegacy := leg[displayName] - - // Skips columns that cannot be uploaded for models.ColumnDefinitionable: - // - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type - if readOnly || displayName == "Title" || isLegacy { - continue - } - - columns = append(columns, cloneColumnDefinitionable(cd)) - } - - newList.SetColumns(columns) - - return newList -} - -// cloneColumnDefinitionable utility function for encapsulating models.ColumnDefinitionable data -// into new object for upload. -func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDefinitionable { - newColumn := models.NewColumnDefinition() - - newColumn.SetAdditionalData(orig.GetAdditionalData()) - newColumn.SetBoolean(orig.GetBoolean()) - newColumn.SetCalculated(orig.GetCalculated()) - newColumn.SetChoice(orig.GetChoice()) - newColumn.SetColumnGroup(orig.GetColumnGroup()) - newColumn.SetContentApprovalStatus(orig.GetContentApprovalStatus()) - newColumn.SetCurrency(orig.GetCurrency()) - newColumn.SetDateTime(orig.GetDateTime()) - newColumn.SetDefaultValue(orig.GetDefaultValue()) - newColumn.SetDescription(orig.GetDescription()) - newColumn.SetDisplayName(orig.GetDisplayName()) - newColumn.SetEnforceUniqueValues(orig.GetEnforceUniqueValues()) - newColumn.SetGeolocation(orig.GetGeolocation()) - newColumn.SetHidden(orig.GetHidden()) - newColumn.SetHyperlinkOrPicture(orig.GetHyperlinkOrPicture()) - newColumn.SetIndexed(orig.GetIndexed()) - newColumn.SetIsDeletable(orig.GetIsDeletable()) - newColumn.SetIsReorderable(orig.GetIsReorderable()) - newColumn.SetIsSealed(orig.GetIsSealed()) - newColumn.SetLookup(orig.GetLookup()) - newColumn.SetName(orig.GetName()) - newColumn.SetNumber(orig.GetNumber()) - newColumn.SetOdataType(orig.GetOdataType()) - newColumn.SetPersonOrGroup(orig.GetPersonOrGroup()) - newColumn.SetPropagateChanges(orig.GetPropagateChanges()) - newColumn.SetReadOnly(orig.GetReadOnly()) - newColumn.SetRequired(orig.GetRequired()) - newColumn.SetSourceColumn(orig.GetSourceColumn()) - newColumn.SetSourceContentType(orig.GetSourceContentType()) - newColumn.SetTerm(orig.GetTerm()) - newColumn.SetText(orig.GetText()) - newColumn.SetThumbnail(orig.GetThumbnail()) - newColumn.SetTypeEscaped(orig.GetTypeEscaped()) - newColumn.SetValidation(orig.GetValidation()) - - return newColumn -} - -// CloneListItem creates a new `SharePoint.ListItem` and stores the original item's -// M365 data into it set fields. -// - https://learn.microsoft.com/en-us/graph/api/resources/listitem?view=graph-rest-1.0 -func CloneListItem(orig models.ListItemable) models.ListItemable { - newItem := models.NewListItem() - newFieldData := retrieveFieldData(orig.GetFields()) - - newItem.SetAdditionalData(orig.GetAdditionalData()) - newItem.SetAnalytics(orig.GetAnalytics()) - newItem.SetContentType(orig.GetContentType()) - newItem.SetCreatedBy(orig.GetCreatedBy()) - newItem.SetCreatedByUser(orig.GetCreatedByUser()) - newItem.SetCreatedDateTime(orig.GetCreatedDateTime()) - newItem.SetDescription(orig.GetDescription()) - // ETag cannot be carried forward - newItem.SetFields(newFieldData) - newItem.SetLastModifiedBy(orig.GetLastModifiedBy()) - newItem.SetLastModifiedByUser(orig.GetLastModifiedByUser()) - newItem.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) - newItem.SetOdataType(orig.GetOdataType()) - // parentReference and SharePointIDs cause error on upload. - // POST Command will link items to the created list. - newItem.SetVersions(orig.GetVersions()) - - return newItem -} - -// retrieveFieldData utility function to clone raw listItem data from the embedded -// additionalData map -// Further details on FieldValueSets: -// - https://learn.microsoft.com/en-us/graph/api/resources/fieldvalueset?view=graph-rest-1.0 -func retrieveFieldData(orig models.FieldValueSetable) models.FieldValueSetable { - fields := models.NewFieldValueSet() - additionalData := make(map[string]any) - fieldData := orig.GetAdditionalData() - - // M365 Book keeping values removed during new Item Creation - // Removed Values: - // -- Prefixes -> @odata.context : absolute path to previous list - // . -> @odata.etag : Embedded link to Prior M365 ID - // -- String Match: Read-Only Fields - // -> id : previous un - for key, value := range fieldData { - if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "@") || - key == "Edit" || key == "Created" || key == "Modified" || - strings.Contains(key, "LookupId") || strings.Contains(key, "ChildCount") || strings.Contains(key, "LinkTitle") { - continue - } - - additionalData[key] = value - } - - fields.SetAdditionalData(additionalData) - - return fields -} diff --git a/src/internal/m365/service/sharepoint/api/serialization_test.go b/src/internal/m365/service/sharepoint/api/serialization_test.go index c79a82284..c52e88b58 100644 --- a/src/internal/m365/service/sharepoint/api/serialization_test.go +++ b/src/internal/m365/service/sharepoint/api/serialization_test.go @@ -4,51 +4,41 @@ import ( "testing" "github.com/alcionai/clues" - kioser "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" - bmodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) type SerializationUnitSuite struct { tester.Suite } -func TestDataSupportSuite(t *testing.T) { +func TestSerializationUnitSuite(t *testing.T) { suite.Run(t, &SerializationUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *SerializationUnitSuite) TestCreateListFromBytes() { +func (suite *SerializationUnitSuite) TestCreateFromBytes() { listBytes, err := spMock.ListBytes("DataSupportSuite") require.NoError(suite.T(), err) tests := []struct { - name string - byteArray []byte - checkError assert.ErrorAssertionFunc - isNil assert.ValueAssertionFunc + name string + byteArray []byte + parseableFunc serialization.ParsableFactory + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc }{ { - name: "empty bytes", - byteArray: make([]byte, 0), - checkError: assert.Error, - isNil: assert.Nil, - }, - { - name: "invalid bytes", - byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), - checkError: assert.Error, - isNil: assert.Nil, - }, - { - name: "Valid List", - byteArray: listBytes, - checkError: assert.NoError, - isNil: assert.NotNil, + name: "Valid List", + byteArray: listBytes, + parseableFunc: models.CreateListFromDiscriminatorValue, + checkError: assert.NoError, + isNil: assert.NotNil, }, } @@ -56,71 +46,9 @@ func (suite *SerializationUnitSuite) TestCreateListFromBytes() { suite.Run(test.name, func() { t := suite.T() - result, err := CreateListFromBytes(test.byteArray) + result, err := createFromBytes(test.byteArray, test.parseableFunc) test.checkError(t, err, clues.ToCore(err)) test.isNil(t, result) }) } } - -func (suite *SerializationUnitSuite) TestCreatePageFromBytes() { - tests := []struct { - name string - checkError assert.ErrorAssertionFunc - isNil assert.ValueAssertionFunc - getBytes func(t *testing.T) []byte - }{ - { - "empty bytes", - assert.Error, - assert.Nil, - func(t *testing.T) []byte { - return make([]byte, 0) - }, - }, - { - "invalid bytes", - assert.Error, - assert.Nil, - func(t *testing.T) []byte { - return []byte("snarf") - }, - }, - { - "Valid Page", - assert.NoError, - assert.NotNil, - func(t *testing.T) []byte { - pg := bmodels.NewSitePage() - title := "Tested" - pg.SetTitle(&title) - pg.SetName(&title) - pg.SetWebUrl(&title) - - writer := kioser.NewJsonSerializationWriter() - err := writer.WriteObjectValue("", pg) - require.NoError(t, err, clues.ToCore(err)) - - byteArray, err := writer.GetSerializedContent() - require.NoError(t, err, clues.ToCore(err)) - - return byteArray - }, - }, - } - - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - result, err := CreatePageFromBytes(test.getBytes(t)) - test.checkError(t, err) - test.isNil(t, result) - if result != nil { - assert.Equal(t, "Tested", *result.GetName(), "name") - assert.Equal(t, "Tested", *result.GetTitle(), "title") - assert.Equal(t, "Tested", *result.GetWebUrl(), "webURL") - } - }) - } -} diff --git a/src/internal/m365/service/sharepoint/mock/mock_test.go b/src/internal/m365/service/sharepoint/mock/mock_test.go index 61590fb9e..778b22bc1 100644 --- a/src/internal/m365/service/sharepoint/mock/mock_test.go +++ b/src/internal/m365/service/sharepoint/mock/mock_test.go @@ -9,8 +9,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type MockSuite struct { @@ -39,7 +40,7 @@ func (suite *MockSuite) TestMockByteHydration() { bytes, err := writer.GetSerializedContent() require.NoError(t, err, clues.ToCore(err)) - _, err = api.CreateListFromBytes(bytes) + _, err = api.BytesToListable(bytes) return err }, @@ -49,7 +50,7 @@ func (suite *MockSuite) TestMockByteHydration() { transformation: func(t *testing.T) error { bytes, err := ListBytes(subject) require.NoError(t, err, clues.ToCore(err)) - _, err = api.CreateListFromBytes(bytes) + _, err = api.BytesToListable(bytes) return err }, }, @@ -57,7 +58,7 @@ func (suite *MockSuite) TestMockByteHydration() { name: "SharePoint: Page", transformation: func(t *testing.T) error { bytes := Page(subject) - _, err := api.CreatePageFromBytes(bytes) + _, err := betaAPI.BytesToSitePageable(bytes) return err }, diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go index c9af42000..bd6b2057d 100644 --- a/src/pkg/services/m365/api/lists.go +++ b/src/pkg/services/m365/api/lists.go @@ -2,14 +2,86 @@ package api import ( "context" + "strings" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/keys" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +var ErrCannotCreateWebTemplateExtension = clues.New("unable to create webTemplateExtension type lists") + +const ( + AttachmentsColumnName = "Attachments" + EditColumnName = "Edit" + ContentTypeColumnName = "ContentType" + CreatedColumnName = "Created" + ModifiedColumnName = "Modified" + AuthorLookupIDColumnName = "AuthorLookupId" + EditorLookupIDColumnName = "EditorLookupId" + + ContentTypeColumnDisplayName = "Content Type" + + AddressFieldName = "address" + CoordinatesFieldName = "coordinates" + DisplayNameFieldName = "displayName" + LocationURIFieldName = "locationUri" + UniqueIDFieldName = "uniqueId" + + CountryOrRegionFieldName = "CountryOrRegion" + StateFieldName = "State" + CityFieldName = "City" + PostalCodeFieldName = "PostalCode" + StreetFieldName = "Street" + GeoLocFieldName = "GeoLoc" + DispNameFieldName = "DispName" + LinkTitleFieldNamePart = "LinkTitle" + ChildCountFieldNamePart = "ChildCount" + + ReadOnlyOrHiddenFieldNamePrefix = "_" + DescoratorFieldNamePrefix = "@" + + WebTemplateExtensionsListTemplateName = "webTemplateExtensionsList" +) + +var addressFieldNames = []string{ + AddressFieldName, + CoordinatesFieldName, + DisplayNameFieldName, + LocationURIFieldName, + UniqueIDFieldName, +} + +var readOnlyAddressFieldNames = []string{ + CountryOrRegionFieldName, + StateFieldName, + CityFieldName, + PostalCodeFieldName, + StreetFieldName, + GeoLocFieldName, + DispNameFieldName, +} + +var legacyColumns = keys.Set{ + AttachmentsColumnName: {}, + EditColumnName: {}, + ContentTypeColumnDisplayName: {}, +} + +var readOnlyFieldNames = keys.Set{ + AttachmentsColumnName: {}, + EditColumnName: {}, + ContentTypeColumnName: {}, + CreatedColumnName: {}, + ModifiedColumnName: {}, + AuthorLookupIDColumnName: {}, + EditorLookupIDColumnName: {}, +} + // --------------------------------------------------------------------------- // controller // --------------------------------------------------------------------------- @@ -62,3 +134,497 @@ func (c Lists) PostDrive( return drive, graph.Wrap(ctx, err, "fetching created documentLibrary").OrNil() } + +// SharePoint lists represent lists on a site. Inherits additional properties from +// baseItem: https://learn.microsoft.com/en-us/graph/api/resources/baseitem?view=graph-rest-1.0 +// The full documentation concerning SharePoint Lists can +// be found at: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 +// Note additional calls are required for the relationships that exist outside of the object properties. + +// GetListById is a utility function to populate a SharePoint.List with objects associated with a given siteID. +// @param siteID the M365 ID that represents the SharePoint Site +// Makes additional calls to retrieve the following relationships: +// - Columns +// - ContentTypes +// - List Items +func (c Lists) GetListByID(ctx context.Context, + siteID, listID string, +) (models.Listable, *details.SharePointInfo, error) { + list, err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Get(ctx, nil) + if err != nil { + return nil, nil, graph.Wrap(ctx, err, "fetching list") + } + + cols, cTypes, lItems, err := c.getListContents(ctx, siteID, listID) + if err != nil { + return nil, nil, graph.Wrap(ctx, err, "getting list contents") + } + + list.SetColumns(cols) + list.SetContentTypes(cTypes) + list.SetItems(lItems) + + return list, ListToSPInfo(list), nil +} + +// getListContents utility function to retrieve associated M365 relationships +// which are not included with the standard List query: +// - Columns, ContentTypes, ListItems +func (c Lists) getListContents(ctx context.Context, siteID, listID string) ( + []models.ColumnDefinitionable, + []models.ContentTypeable, + []models.ListItemable, + error, +) { + cols, err := c.GetListColumns(ctx, siteID, listID, CallConfig{}) + if err != nil { + return nil, nil, nil, err + } + + cTypes, err := c.GetContentTypes(ctx, siteID, listID, CallConfig{}) + if err != nil { + return nil, nil, nil, err + } + + for i := 0; i < len(cTypes); i++ { + columnLinks, err := c.GetColumnLinks(ctx, siteID, listID, ptr.Val(cTypes[i].GetId()), CallConfig{}) + if err != nil { + return nil, nil, nil, err + } + + cTypes[i].SetColumnLinks(columnLinks) + + cTypeColumns, err := c.GetCTypesColumns(ctx, siteID, listID, ptr.Val(cTypes[i].GetId()), CallConfig{}) + if err != nil { + return nil, nil, nil, err + } + + cTypes[i].SetColumns(cTypeColumns) + } + + lItems, err := c.GetListItems(ctx, siteID, listID, CallConfig{}) + if err != nil { + return nil, nil, nil, err + } + + for _, li := range lItems { + fields, err := c.getListItemFields(ctx, siteID, listID, ptr.Val(li.GetId())) + if err != nil { + return nil, nil, nil, err + } + + li.SetFields(fields) + } + + return cols, cTypes, lItems, nil +} + +func (c Lists) PostList( + ctx context.Context, + siteID string, + listName string, + oldListByteArray []byte, +) (models.Listable, error) { + newListName := listName + + oldList, err := BytesToListable(oldListByteArray) + if err != nil { + return nil, clues.WrapWC(ctx, err, "generating list from stored bytes") + } + + // the input listName is of format: destinationName_listID + // here we replace listID with displayName of list generated from stored bytes + if name, ok := ptr.ValOK(oldList.GetDisplayName()); ok { + nameParts := strings.Split(listName, "_") + if len(nameParts) > 0 { + nameParts[len(nameParts)-1] = name + newListName = strings.Join(nameParts, "_") + } + } + + // this ensure all columns, contentTypes are set to the newList + newList := ToListable(oldList, newListName) + + if newList != nil && + newList.GetList() != nil && + ptr.Val(newList.GetList().GetTemplate()) == WebTemplateExtensionsListTemplateName { + return nil, clues.StackWC(ctx, ErrCannotCreateWebTemplateExtension) + } + + // Restore to List base to M365 back store + restoredList, err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + Post(ctx, newList, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "creating list") + } + + listItems := make([]models.ListItemable, 0) + + for _, itm := range oldList.GetItems() { + temp := CloneListItem(itm) + listItems = append(listItems, temp) + } + + err = c.PostListItems( + ctx, + siteID, + ptr.Val(restoredList.GetId()), + listItems) + if err == nil { + restoredList.SetItems(listItems) + return restoredList, nil + } + + // [TODO](hitesh) double check if we need to: + // 1. rollback the entire list + // 2. restore as much list items possible and add recoverables to fault bus + // rollback list creation + err = c.DeleteList(ctx, siteID, ptr.Val(restoredList.GetId())) + + return nil, graph.Wrap(ctx, err, "deleting restored list after items creation failure").OrNil() +} + +func (c Lists) PostListItems( + ctx context.Context, + siteID, listID string, + listItems []models.ListItemable, +) error { + for _, lItem := range listItems { + _, err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Items(). + Post(ctx, lItem, nil) + if err != nil { + return graph.Wrap(ctx, err, "creating item in list") + } + } + + return nil +} + +func (c Lists) DeleteList( + ctx context.Context, + siteID, listID string, +) error { + err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Delete(ctx, nil) + + return graph.Wrap(ctx, err, "deleting list").OrNil() +} + +func BytesToListable(bytes []byte) (models.Listable, error) { + parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue) + if err != nil { + return nil, clues.Wrap(err, "deserializing bytes to sharepoint list") + } + + list := parsable.(models.Listable) + + return list, nil +} + +// ToListable utility function to encapsulate stored data for restoration. +// New Listable omits trackable fields such as `id` or `ETag` and other read-only +// objects that are prevented upon upload. Additionally, read-Only columns are +// not attached in this method. +// ListItems are not included in creation of new list, and have to be restored +// in separate call. +func ToListable(orig models.Listable, displayName string) models.Listable { + newList := models.NewList() + + newList.SetContentTypes(orig.GetContentTypes()) + newList.SetCreatedBy(orig.GetCreatedBy()) + newList.SetCreatedByUser(orig.GetCreatedByUser()) + newList.SetCreatedDateTime(orig.GetCreatedDateTime()) + newList.SetDescription(orig.GetDescription()) + newList.SetDisplayName(&displayName) + newList.SetLastModifiedBy(orig.GetLastModifiedBy()) + newList.SetLastModifiedByUser(orig.GetLastModifiedByUser()) + newList.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) + newList.SetList(orig.GetList()) + newList.SetOdataType(orig.GetOdataType()) + newList.SetParentReference(orig.GetParentReference()) + + columns := make([]models.ColumnDefinitionable, 0) + + for _, cd := range orig.GetColumns() { + var ( + displayName string + readOnly bool + ) + + if name, ok := ptr.ValOK(cd.GetDisplayName()); ok { + displayName = name + } + + if ro, ok := ptr.ValOK(cd.GetReadOnly()); ok { + readOnly = ro + } + + // Skips columns that cannot be uploaded for models.ColumnDefinitionable: + // - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type + if readOnly || displayName == "Title" || legacyColumns.HasKey(displayName) { + continue + } + + columns = append(columns, cloneColumnDefinitionable(cd)) + } + + newList.SetColumns(columns) + + return newList +} + +// cloneColumnDefinitionable utility function for encapsulating models.ColumnDefinitionable data +// into new object for upload. +func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDefinitionable { + newColumn := models.NewColumnDefinition() + + // column attributes + newColumn.SetName(orig.GetName()) + newColumn.SetOdataType(orig.GetOdataType()) + newColumn.SetPropagateChanges(orig.GetPropagateChanges()) + newColumn.SetReadOnly(orig.GetReadOnly()) + newColumn.SetRequired(orig.GetRequired()) + newColumn.SetAdditionalData(orig.GetAdditionalData()) + newColumn.SetDescription(orig.GetDescription()) + newColumn.SetDisplayName(orig.GetDisplayName()) + newColumn.SetSourceColumn(orig.GetSourceColumn()) + newColumn.SetSourceContentType(orig.GetSourceContentType()) + newColumn.SetHidden(orig.GetHidden()) + newColumn.SetIndexed(orig.GetIndexed()) + newColumn.SetIsDeletable(orig.GetIsDeletable()) + newColumn.SetIsReorderable(orig.GetIsReorderable()) + newColumn.SetIsSealed(orig.GetIsSealed()) + newColumn.SetTypeEscaped(orig.GetTypeEscaped()) + newColumn.SetColumnGroup(orig.GetColumnGroup()) + newColumn.SetEnforceUniqueValues(orig.GetEnforceUniqueValues()) + + // column types + setColumnType(newColumn, orig) + + // Requires nil checks to avoid Graph error: 'General exception while processing' + defaultValue := orig.GetDefaultValue() + if defaultValue != nil { + newColumn.SetDefaultValue(defaultValue) + } + + validation := orig.GetValidation() + if validation != nil { + newColumn.SetValidation(validation) + } + + return newColumn +} + +func setColumnType(newColumn *models.ColumnDefinition, orig models.ColumnDefinitionable) { + switch { + case orig.GetText() != nil: + newColumn.SetText(orig.GetText()) + case orig.GetBoolean() != nil: + newColumn.SetBoolean(orig.GetBoolean()) + case orig.GetCalculated() != nil: + newColumn.SetCalculated(orig.GetCalculated()) + case orig.GetChoice() != nil: + newColumn.SetChoice(orig.GetChoice()) + case orig.GetContentApprovalStatus() != nil: + newColumn.SetContentApprovalStatus(orig.GetContentApprovalStatus()) + case orig.GetCurrency() != nil: + newColumn.SetCurrency(orig.GetCurrency()) + case orig.GetDateTime() != nil: + newColumn.SetDateTime(orig.GetDateTime()) + case orig.GetGeolocation() != nil: + newColumn.SetGeolocation(orig.GetGeolocation()) + case orig.GetHyperlinkOrPicture() != nil: + newColumn.SetHyperlinkOrPicture(orig.GetHyperlinkOrPicture()) + case orig.GetNumber() != nil: + newColumn.SetNumber(orig.GetNumber()) + case orig.GetLookup() != nil: + newColumn.SetLookup(orig.GetLookup()) + case orig.GetThumbnail() != nil: + newColumn.SetThumbnail(orig.GetThumbnail()) + case orig.GetTerm() != nil: + newColumn.SetTerm(orig.GetTerm()) + case orig.GetPersonOrGroup() != nil: + newColumn.SetPersonOrGroup(orig.GetPersonOrGroup()) + default: + newColumn.SetText(models.NewTextColumn()) + } +} + +// CloneListItem creates a new `SharePoint.ListItem` and stores the original item's +// M365 data into it set fields. +// - https://learn.microsoft.com/en-us/graph/api/resources/listitem?view=graph-rest-1.0 +func CloneListItem(orig models.ListItemable) models.ListItemable { + newItem := models.NewListItem() + + // list item data + newFieldData := retrieveFieldData(orig.GetFields()) + newItem.SetFields(newFieldData) + + // list item attributes + newItem.SetAdditionalData(orig.GetAdditionalData()) + newItem.SetDescription(orig.GetDescription()) + newItem.SetCreatedBy(orig.GetCreatedBy()) + newItem.SetCreatedDateTime(orig.GetCreatedDateTime()) + newItem.SetLastModifiedBy(orig.GetLastModifiedBy()) + newItem.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) + newItem.SetOdataType(orig.GetOdataType()) + newItem.SetAnalytics(orig.GetAnalytics()) + newItem.SetContentType(orig.GetContentType()) + newItem.SetVersions(orig.GetVersions()) + + // Requires nil checks to avoid Graph error: 'Invalid request' + lastCreatedByUser := orig.GetCreatedByUser() + if lastCreatedByUser != nil { + newItem.SetCreatedByUser(lastCreatedByUser) + } + + lastModifiedByUser := orig.GetLastModifiedByUser() + if lastCreatedByUser != nil { + newItem.SetLastModifiedByUser(lastModifiedByUser) + } + + return newItem +} + +// retrieveFieldData utility function to clone raw listItem data from the embedded +// additionalData map +// Further documentation on FieldValueSets: +// - https://learn.microsoft.com/en-us/graph/api/resources/fieldvalueset?view=graph-rest-1.0 +func retrieveFieldData(orig models.FieldValueSetable) models.FieldValueSetable { + fields := models.NewFieldValueSet() + additionalData := filterAdditionalData(orig) + + retainPrimaryAddressField(additionalData) + + fields.SetAdditionalData(additionalData) + + return fields +} + +func filterAdditionalData(orig models.FieldValueSetable) map[string]any { + if orig == nil { + return make(map[string]any) + } + + fieldData := orig.GetAdditionalData() + filteredData := make(map[string]any) + + for key, value := range fieldData { + if shouldFilterField(key, value) { + continue + } + + filteredData[key] = value + } + + return filteredData +} + +func shouldFilterField(key string, value any) bool { + return readOnlyFieldNames.HasKey(key) || + strings.HasPrefix(key, ReadOnlyOrHiddenFieldNamePrefix) || + strings.HasPrefix(key, DescoratorFieldNamePrefix) || + strings.Contains(key, LinkTitleFieldNamePart) || + strings.Contains(key, ChildCountFieldNamePart) +} + +func retainPrimaryAddressField(additionalData map[string]any) { + if !hasAddressFields(additionalData) { + return + } + + for _, k := range readOnlyAddressFieldNames { + delete(additionalData, k) + } +} + +func hasAddressFields(additionalData map[string]any) bool { + if !keys.HasKeys(additionalData, readOnlyAddressFieldNames...) { + return false + } + + for _, value := range additionalData { + nestedFields, ok := value.(map[string]any) + if !ok || keys.HasKeys(nestedFields, GeoLocFieldName) { + continue + } + + if keys.HasKeys(nestedFields, addressFieldNames...) { + return true + } + } + + return false +} + +func (c Lists) getListItemFields( + ctx context.Context, + siteID, listID, itemID string, +) (models.FieldValueSetable, error) { + prefix := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Items(). + ByListItemId(itemID) + + fields, err := prefix.Fields().Get(ctx, nil) + if err != nil { + return nil, err + } + + return fields, nil +} + +// ListToSPInfo translates models.Listable metadata into searchable content +// List documentation: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 +func ListToSPInfo(lst models.Listable) *details.SharePointInfo { + var ( + name = ptr.Val(lst.GetDisplayName()) + webURL = ptr.Val(lst.GetWebUrl()) + created = ptr.Val(lst.GetCreatedDateTime()) + modified = ptr.Val(lst.GetLastModifiedDateTime()) + count = len(lst.GetItems()) + ) + + template := "" + if lst.GetList() != nil { + template = ptr.Val(lst.GetList().GetTemplate()) + } + + return &details.SharePointInfo{ + ItemType: details.SharePointList, + Modified: modified, + List: &details.ListInfo{ + Name: name, + ItemCount: int64(count), + Template: template, + Created: created, + Modified: modified, + WebURL: webURL, + }, + } +} diff --git a/src/pkg/services/m365/api/lists_pager.go b/src/pkg/services/m365/api/lists_pager.go new file mode 100644 index 000000000..29d389b5d --- /dev/null +++ b/src/pkg/services/m365/api/lists_pager.go @@ -0,0 +1,455 @@ +package api + +import ( + "context" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/sites" + + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" +) + +// --------------------------------------------------------------------------- +// lists pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.Listable] = &listsPageCtrl{} + +type listsPageCtrl struct { + siteID string + gs graph.Servicer + builder *sites.ItemListsRequestBuilder + options *sites.ItemListsRequestBuilderGetRequestConfiguration +} + +func (p *listsPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *listsPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.Listable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *listsPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewListsPager( + siteID string, + cc CallConfig, +) *listsPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists() + + options := &sites.ItemListsRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &listsPageCtrl{ + siteID: siteID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetLists fetches all lists in the site. +func (c Lists) GetLists( + ctx context.Context, + siteID string, + cc CallConfig, +) ([]models.Listable, error) { + pager := c.NewListsPager(siteID, cc) + items, err := pagers.BatchEnumerateItems[models.Listable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// list items pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.ListItemable] = &listItemsPageCtrl{} + +type listItemsPageCtrl struct { + siteID string + listID string + gs graph.Servicer + builder *sites.ItemListsItemItemsRequestBuilder + options *sites.ItemListsItemItemsRequestBuilderGetRequestConfiguration +} + +func (p *listItemsPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsItemItemsRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *listItemsPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.ListItemable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *listItemsPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewListItemsPager( + siteID string, + listID string, + cc CallConfig, +) *listItemsPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Items() + + options := &sites.ItemListsItemItemsRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsItemItemsRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &listItemsPageCtrl{ + siteID: siteID, + listID: listID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetListItems fetches all list items in the list. +func (c Lists) GetListItems( + ctx context.Context, + siteID string, + listID string, + cc CallConfig, +) ([]models.ListItemable, error) { + pager := c.NewListItemsPager(siteID, listID, cc) + items, err := pagers.BatchEnumerateItems[models.ListItemable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// columns pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.ColumnDefinitionable] = &columnsPageCtrl{} + +type columnsPageCtrl struct { + siteID string + listID string + gs graph.Servicer + builder *sites.ItemListsItemColumnsRequestBuilder + options *sites.ItemListsItemColumnsRequestBuilderGetRequestConfiguration +} + +func (p *columnsPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsItemColumnsRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *columnsPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.ColumnDefinitionable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *columnsPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewColumnsPager( + siteID string, + listID string, + cc CallConfig, +) *columnsPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Columns() + + options := &sites.ItemListsItemColumnsRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsItemColumnsRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &columnsPageCtrl{ + siteID: siteID, + listID: listID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetListColumns fetches all list columns in the list. +func (c Lists) GetListColumns( + ctx context.Context, + siteID string, + listID string, + cc CallConfig, +) ([]models.ColumnDefinitionable, error) { + pager := c.NewColumnsPager(siteID, listID, cc) + items, err := pagers.BatchEnumerateItems[models.ColumnDefinitionable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// content types pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.ContentTypeable] = &contentTypesPageCtrl{} + +type contentTypesPageCtrl struct { + siteID string + listID string + gs graph.Servicer + builder *sites.ItemListsItemContentTypesRequestBuilder + options *sites.ItemListsItemContentTypesRequestBuilderGetRequestConfiguration +} + +func (p *contentTypesPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsItemContentTypesRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *contentTypesPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.ContentTypeable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *contentTypesPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewContentTypesPager( + siteID string, + listID string, + cc CallConfig, +) *contentTypesPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + ContentTypes() + + options := &sites.ItemListsItemContentTypesRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsItemContentTypesRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &contentTypesPageCtrl{ + siteID: siteID, + listID: listID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetContentTypes fetches all content types in the list. +func (c Lists) GetContentTypes( + ctx context.Context, + siteID string, + listID string, + cc CallConfig, +) ([]models.ContentTypeable, error) { + pager := c.NewContentTypesPager(siteID, listID, cc) + items, err := pagers.BatchEnumerateItems[models.ContentTypeable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// content types columns pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.ColumnDefinitionable] = &cTypesColumnsPageCtrl{} + +type cTypesColumnsPageCtrl struct { + siteID string + listID string + contentTypeID string + gs graph.Servicer + builder *sites.ItemListsItemContentTypesItemColumnsRequestBuilder + options *sites.ItemListsItemContentTypesItemColumnsRequestBuilderGetRequestConfiguration +} + +func (p *cTypesColumnsPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsItemContentTypesItemColumnsRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *cTypesColumnsPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.ColumnDefinitionable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *cTypesColumnsPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewCTypesColumnsPager( + siteID string, + listID string, + contentTypeID string, + cc CallConfig, +) *cTypesColumnsPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + ContentTypes(). + ByContentTypeId(contentTypeID). + Columns() + + options := &sites.ItemListsItemContentTypesItemColumnsRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsItemContentTypesItemColumnsRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &cTypesColumnsPageCtrl{ + siteID: siteID, + listID: listID, + contentTypeID: contentTypeID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetCTypesColumns fetches all columns in the content type. +func (c Lists) GetCTypesColumns( + ctx context.Context, + siteID string, + listID string, + contentTypeID string, + cc CallConfig, +) ([]models.ColumnDefinitionable, error) { + pager := c.NewCTypesColumnsPager(siteID, listID, contentTypeID, cc) + items, err := pagers.BatchEnumerateItems[models.ColumnDefinitionable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// column links pager +// --------------------------------------------------------------------------- + +var _ pagers.NonDeltaHandler[models.ColumnLinkable] = &columnLinksPageCtrl{} + +type columnLinksPageCtrl struct { + siteID string + listID string + contentTypeID string + gs graph.Servicer + builder *sites.ItemListsItemContentTypesItemColumnLinksRequestBuilder + options *sites.ItemListsItemContentTypesItemColumnLinksRequestBuilderGetRequestConfiguration +} + +func (p *columnLinksPageCtrl) SetNextLink(nextLink string) { + p.builder = sites.NewItemListsItemContentTypesItemColumnLinksRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (p *columnLinksPageCtrl) GetPage( + ctx context.Context, +) (pagers.NextLinkValuer[models.ColumnLinkable], error) { + resp, err := p.builder.Get(ctx, p.options) + return resp, graph.Stack(ctx, err).OrNil() +} + +func (p *columnLinksPageCtrl) ValidModTimes() bool { + return true +} + +func (c Lists) NewColumnLinksPager( + siteID string, + listID string, + contentTypeID string, + cc CallConfig, +) *columnLinksPageCtrl { + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + ContentTypes(). + ByContentTypeId(contentTypeID). + ColumnLinks() + + options := &sites.ItemListsItemContentTypesItemColumnLinksRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemListsItemContentTypesItemColumnLinksRequestBuilderGetQueryParameters{}, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)), + } + + if len(cc.Select) > 0 { + options.QueryParameters.Select = cc.Select + } + + return &columnLinksPageCtrl{ + siteID: siteID, + listID: listID, + contentTypeID: contentTypeID, + builder: builder, + gs: c.Stable, + options: options, + } +} + +// GetColumnLinks fetches all column links in the content type. +func (c Lists) GetColumnLinks( + ctx context.Context, + siteID string, + listID string, + contentTypeID string, + cc CallConfig, +) ([]models.ColumnLinkable, error) { + pager := c.NewColumnLinksPager(siteID, listID, contentTypeID, cc) + items, err := pagers.BatchEnumerateItems[models.ColumnLinkable](ctx, pager) + + return items, graph.Stack(ctx, err).OrNil() +} diff --git a/src/pkg/services/m365/api/lists_pager_test.go b/src/pkg/services/m365/api/lists_pager_test.go new file mode 100644 index 000000000..f7d7fa9dd --- /dev/null +++ b/src/pkg/services/m365/api/lists_pager_test.go @@ -0,0 +1,350 @@ +package api + +import ( + "context" + "testing" + + "github.com/alcionai/clues" + "github.com/h2non/gock" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + graphTD "github.com/alcionai/corso/src/pkg/services/m365/api/graph/testdata" +) + +type ListsPagerIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestListsPagerIntgSuite(t *testing.T) { + suite.Run(t, &ListsPagerIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *ListsPagerIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *ListsPagerIntgSuite) TestEnumerateLists_withAssociatedRelationships() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + + listID = "fake-list-id" + siteID = suite.its.site.id + textColumnDefID = "fake-text-column-id" + textColumnDefName = "itemName" + numColumnDefID = "fake-num-column-id" + numColumnDefName = "itemSize" + colLinkID = "fake-collink-id" + cTypeID = "fake-ctype-id" + listItemID = "fake-list-item-id" + + fieldsData = map[string]any{ + "itemName": "item1", + "itemSize": 4, + } + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + defer gock.Off() + + suite.setStubListAndItsRelationShip(listID, + siteID, + textColumnDefID, + textColumnDefName, + numColumnDefID, + numColumnDefName, + colLinkID, + cTypeID, + listItemID, + fieldsData) + + lists, err := ac.GetLists(ctx, suite.its.site.id, CallConfig{}) + require.NoError(t, err) + require.Equal(t, 1, len(lists)) + + for _, list := range lists { + suite.testEnumerateListItems(ctx, list, listItemID, fieldsData) + suite.testEnumerateColumns(ctx, list, textColumnDefID) + suite.testEnumerateContentTypes(ctx, list, cTypeID, colLinkID, numColumnDefID) + } +} + +func (suite *ListsPagerIntgSuite) testEnumerateListItems( + ctx context.Context, + list models.Listable, + expectedListItemID string, + setFieldsData map[string]any, +) []models.ListItemable { + var listItems []models.ListItemable + + suite.Run("list item", func() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + err error + ) + + listItems, err = ac.GetListItems(ctx, suite.its.site.id, *list.GetId(), CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, 1, len(listItems)) + + for _, li := range listItems { + assert.Equal(t, expectedListItemID, *li.GetId()) + + fields := li.GetFields() + require.NotEmpty(t, fields) + + fieldsData := fields.GetAdditionalData() + expectedItemName := setFieldsData["itemName"].(string) + actualItemName := ptr.Val(fieldsData["itemName"].(*string)) + expectedItemSize := setFieldsData["itemSize"].(int) + actualItemSize := int(ptr.Val(fieldsData["itemSize"].(*float64))) + + assert.Equal(t, expectedItemName, actualItemName) + assert.Equal(t, expectedItemSize, actualItemSize) + } + }) + + return listItems +} + +func (suite *ListsPagerIntgSuite) testEnumerateColumns( + ctx context.Context, + list models.Listable, + expectedColumnID string, +) []models.ColumnDefinitionable { + var columns []models.ColumnDefinitionable + + suite.Run("list columns", func() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + err error + ) + + columns, err = ac.GetListColumns(ctx, suite.its.site.id, *list.GetId(), CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, 1, len(columns)) + + for _, c := range columns { + assert.Equal(suite.T(), expectedColumnID, *c.GetId()) + } + }) + + return columns +} + +func (suite *ListsPagerIntgSuite) testEnumerateContentTypes( + ctx context.Context, + list models.Listable, + expectedCTypeID, + expectedColLinkID, + expectedCTypeColID string, +) []models.ContentTypeable { + var cTypes []models.ContentTypeable + + suite.Run("content type", func() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + err error + ) + + cTypes, err = ac.GetContentTypes(ctx, suite.its.site.id, *list.GetId(), CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, 1, len(cTypes)) + + for _, ct := range cTypes { + assert.Equal(suite.T(), expectedCTypeID, *ct.GetId()) + + suite.testEnumerateColumnLinks(ctx, list, ct, expectedColLinkID) + suite.testEnumerateCTypeColumns(ctx, list, ct, expectedCTypeColID) + } + }) + + return cTypes +} + +func (suite *ListsPagerIntgSuite) testEnumerateColumnLinks( + ctx context.Context, + list models.Listable, + cType models.ContentTypeable, + expectedColLinkID string, +) []models.ColumnLinkable { + var colLinks []models.ColumnLinkable + + suite.Run("column links", func() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + err error + ) + + colLinks, err = ac.GetColumnLinks(ctx, suite.its.site.id, *list.GetId(), *cType.GetId(), CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, 1, len(colLinks)) + + for _, cl := range colLinks { + assert.Equal(suite.T(), expectedColLinkID, *cl.GetId()) + } + }) + + return colLinks +} + +func (suite *ListsPagerIntgSuite) testEnumerateCTypeColumns( + ctx context.Context, + list models.Listable, + cType models.ContentTypeable, + expectedCTypeColID string, +) []models.ColumnDefinitionable { + var cTypeCols []models.ColumnDefinitionable + + suite.Run("ctype columns", func() { + var ( + t = suite.T() + ac = suite.its.gockAC.Lists() + err error + ) + + cTypeCols, err = ac.GetCTypesColumns(ctx, suite.its.site.id, *list.GetId(), *cType.GetId(), CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, 1, len(cTypeCols)) + + for _, c := range cTypeCols { + assert.Equal(suite.T(), expectedCTypeColID, *c.GetId()) + } + }) + + return cTypeCols +} + +func (suite *ListsPagerIntgSuite) setStubListAndItsRelationShip( + listID, + siteID, + textColumnDefID, + textColumnDefName, + numColumnDefID, + numColumnDefName, + colLinkID, + cTypeID, + listItemID string, + fieldsData map[string]any, +) { + list := models.NewList() + list.SetId(&listID) + + listCol := models.NewListCollectionResponse() + listCol.SetValue([]models.Listable{list}) + + txtColumnDef := models.NewColumnDefinition() + txtColumnDef.SetId(&textColumnDefID) + txtColumnDef.SetName(&textColumnDefName) + + textColumn := models.NewTextColumn() + txtColumnDef.SetText(textColumn) + + columnDefCol := models.NewColumnDefinitionCollectionResponse() + columnDefCol.SetValue([]models.ColumnDefinitionable{txtColumnDef}) + + numColumnDef := models.NewColumnDefinition() + numColumnDef.SetId(&numColumnDefID) + numColumnDef.SetName(&numColumnDefName) + + numColumn := models.NewNumberColumn() + numColumnDef.SetNumber(numColumn) + + columnDefCol2 := models.NewColumnDefinitionCollectionResponse() + columnDefCol2.SetValue([]models.ColumnDefinitionable{numColumnDef}) + + colLink := models.NewColumnLink() + colLink.SetId(&colLinkID) + + colLinkCol := models.NewColumnLinkCollectionResponse() + colLinkCol.SetValue([]models.ColumnLinkable{colLink}) + + cTypes := models.NewContentType() + cTypes.SetId(&cTypeID) + + cTypesCol := models.NewContentTypeCollectionResponse() + cTypesCol.SetValue([]models.ContentTypeable{cTypes}) + + fields := models.NewFieldValueSet() + fields.SetAdditionalData(fieldsData) + + listItem := models.NewListItem() + listItem.SetId(&listItemID) + listItem.SetFields(fields) + + listItemCol := models.NewListItemCollectionResponse() + listItemCol.SetValue([]models.ListItemable{listItem}) + + interceptV1Path( + "sites", + siteID, + "lists"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), listCol)) + + interceptV1Path( + "sites", + siteID, + "lists", + listID, + "columns"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), columnDefCol)) + + interceptV1Path( + "sites", + siteID, + "lists", + listID, + "items"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), listItemCol)) + + interceptV1Path( + "sites", + siteID, + "lists", + listID, + "contentTypes"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), cTypesCol)) + + interceptV1Path( + "sites", + siteID, + "lists", + listID, + "contentTypes", + cTypeID, + "columns"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), columnDefCol2)) + + interceptV1Path( + "sites", + siteID, + "lists", + listID, + "contentTypes", + cTypeID, + "columnLinks"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), colLinkCol)) +} diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go index a76c30636..9fcfbdf02 100644 --- a/src/pkg/services/m365/api/lists_test.go +++ b/src/pkg/services/m365/api/lists_test.go @@ -2,19 +2,500 @@ package api import ( "testing" + "time" "github.com/alcionai/clues" + "github.com/h2non/gock" + kjson "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + graphTD "github.com/alcionai/corso/src/pkg/services/m365/api/graph/testdata" ) +type ListsUnitSuite struct { + tester.Suite +} + +func TestListsUnitSuite(t *testing.T) { + suite.Run(t, &ListsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ListsUnitSuite) TestSharePointInfo() { + tests := []struct { + name string + listAndDeets func() (models.Listable, *details.SharePointInfo) + }{ + { + name: "Empty List", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + i := &details.SharePointInfo{ItemType: details.SharePointList} + return models.NewList(), i + }, + }, { + name: "Only Name", + listAndDeets: func() (models.Listable, *details.SharePointInfo) { + aTitle := "Whole List" + listing := models.NewList() + listing.SetDisplayName(&aTitle) + + li := models.NewListItem() + li.SetId(ptr.To("listItem1")) + + listing.SetItems([]models.ListItemable{li}) + i := &details.SharePointInfo{ + ItemType: details.SharePointList, + List: &details.ListInfo{ + Name: aTitle, + ItemCount: 1, + }, + } + + return listing, i + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + list, expected := test.listAndDeets() + info := ListToSPInfo(list) + assert.Equal(t, expected.ItemType, info.ItemType) + assert.Equal(t, expected.ItemName, info.ItemName) + assert.Equal(t, expected.WebURL, info.WebURL) + if expected.List != nil { + assert.Equal(t, expected.List.ItemCount, info.List.ItemCount) + } + }) + } +} + +func (suite *ListsUnitSuite) TestBytesToListable() { + listBytes, err := spMock.ListBytes("DataSupportSuite") + require.NoError(suite.T(), err) + + tests := []struct { + name string + byteArray []byte + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + }{ + { + name: "empty bytes", + byteArray: make([]byte, 0), + checkError: assert.Error, + isNil: assert.Nil, + }, + { + name: "invalid bytes", + byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), + checkError: assert.Error, + isNil: assert.Nil, + }, + { + name: "Valid List", + byteArray: listBytes, + checkError: assert.NoError, + isNil: assert.NotNil, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + result, err := BytesToListable(test.byteArray) + test.checkError(t, err, clues.ToCore(err)) + test.isNil(t, result) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_GetValidation() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + expect assert.ValueAssertionFunc + }{ + { + name: "column validation not set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + + return cd + }, + expect: assert.Nil, + }, + { + name: "column validation set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + colValidation := models.NewColumnValidation() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + cd.SetValidation(colValidation) + + return cd + }, + expect: assert.NotNil, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + + test.expect(t, newCd.GetValidation()) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_GetDefaultValue() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + expect func(t *testing.T, cd models.ColumnDefinitionable) + }{ + { + name: "column default value not set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + + return cd + }, + expect: func(t *testing.T, cd models.ColumnDefinitionable) { + assert.Nil(t, cd.GetDefaultValue()) + }, + }, + { + name: "column default value set", + getOrig: func() models.ColumnDefinitionable { + defaultVal := "some-val" + + textColumn := models.NewTextColumn() + + colDefaultVal := models.NewDefaultColumnValue() + colDefaultVal.SetValue(ptr.To(defaultVal)) + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + cd.SetDefaultValue(colDefaultVal) + + return cd + }, + expect: func(t *testing.T, cd models.ColumnDefinitionable) { + assert.NotNil(t, cd.GetDefaultValue()) + assert.Equal(t, "some-val", ptr.Val(cd.GetDefaultValue().GetValue())) + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + test.expect(t, newCd) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_ColumnType() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + checkFn func(models.ColumnDefinitionable) bool + }{ + { + name: "column type should be number", + getOrig: func() models.ColumnDefinitionable { + numColumn := models.NewNumberColumn() + + cd := models.NewColumnDefinition() + cd.SetNumber(numColumn) + + return cd + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetNumber() != nil + }, + }, + { + name: "column type should be person or group", + getOrig: func() models.ColumnDefinitionable { + pgColumn := models.NewPersonOrGroupColumn() + + cd := models.NewColumnDefinition() + cd.SetPersonOrGroup(pgColumn) + + return cd + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetPersonOrGroup() != nil + }, + }, + { + name: "column type should default to text", + getOrig: func() models.ColumnDefinitionable { + return models.NewColumnDefinition() + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetText() != nil + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + assert.True(t, test.checkFn(newCd)) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() { + listName := "test-list" + textColumnName := "ItemName" + textColumnDisplayName := "Item Name" + titleColumnName := "Title" + titleColumnDisplayName := "Title" + readOnlyColumnName := "TestColumn" + readOnlyColumnDisplayName := "Test Column" + + contentTypeCd := models.NewColumnDefinition() + contentTypeCd.SetName(ptr.To(ContentTypeColumnName)) + contentTypeCd.SetDisplayName(ptr.To(ContentTypeColumnDisplayName)) + + attachmentCd := models.NewColumnDefinition() + attachmentCd.SetName(ptr.To(AttachmentsColumnName)) + attachmentCd.SetDisplayName(ptr.To(AttachmentsColumnName)) + + editCd := models.NewColumnDefinition() + editCd.SetName(ptr.To(EditColumnName)) + editCd.SetDisplayName(ptr.To(EditColumnName)) + + textCol := models.NewTextColumn() + titleCol := models.NewTextColumn() + roCol := models.NewTextColumn() + + textCd := models.NewColumnDefinition() + textCd.SetName(ptr.To(textColumnName)) + textCd.SetDisplayName(ptr.To(textColumnDisplayName)) + textCd.SetText(textCol) + + titleCd := models.NewColumnDefinition() + titleCd.SetName(ptr.To(titleColumnName)) + titleCd.SetDisplayName(ptr.To(titleColumnDisplayName)) + titleCd.SetText(titleCol) + + roCd := models.NewColumnDefinition() + roCd.SetName(ptr.To(readOnlyColumnName)) + roCd.SetDisplayName(ptr.To(readOnlyColumnDisplayName)) + roCd.SetText(roCol) + roCd.SetReadOnly(ptr.To(true)) + + tests := []struct { + name string + getList func() *models.List + length int + }{ + { + name: "all legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + }) + return lst + }, + length: 0, + }, + { + name: "title and legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + titleCd, + }) + return lst + }, + length: 0, + }, + { + name: "readonly and legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + roCd, + }) + return lst + }, + length: 0, + }, + { + name: "legacy and a text column", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + textCd, + }) + return lst + }, + length: 1, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + clonedList := ToListable(test.getList(), listName) + require.NotEmpty(t, clonedList) + + cols := clonedList.GetColumns() + assert.Len(t, cols, test.length) + }) + } +} + +func (suite *ListsUnitSuite) TestFieldValueSetable() { + t := suite.T() + + additionalData := map[string]any{ + DescoratorFieldNamePrefix + "odata.etag": "14fe12b2-e180-49f7-8fc3-5936f3dcf5d2,1", + ReadOnlyOrHiddenFieldNamePrefix + "UIVersionString": "1.0", + AuthorLookupIDColumnName: "6", + EditorLookupIDColumnName: "6", + "Item" + ChildCountFieldNamePart: "0", + "Folder" + ChildCountFieldNamePart: "0", + ModifiedColumnName: "2023-12-13T15:47:51Z", + CreatedColumnName: "2023-12-13T15:47:51Z", + EditColumnName: "", + LinkTitleFieldNamePart + "NoMenu": "Person1", + } + + origFs := models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs := retrieveFieldData(origFs) + fsAdditionalData := fs.GetAdditionalData() + assert.Empty(t, fsAdditionalData) + + additionalData["itemName"] = "item-1" + origFs = models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs = retrieveFieldData(origFs) + fsAdditionalData = fs.GetAdditionalData() + assert.NotEmpty(t, fsAdditionalData) + + val, ok := fsAdditionalData["itemName"] + assert.True(t, ok) + assert.Equal(t, "item-1", val) +} + +func (suite *ListsUnitSuite) TestFieldValueSetable_Location() { + t := suite.T() + + additionalData := map[string]any{ + "MyAddress": map[string]any{ + AddressFieldName: map[string]any{ + "city": "Tagaytay", + "countryOrRegion": "Philippines", + "postalCode": "4120", + "state": "Calabarzon", + "street": "Prime Residences CityLand 1852", + }, + CoordinatesFieldName: map[string]any{ + "latitude": "14.1153", + "longitude": "120.962", + }, + DisplayNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + LocationURIFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + UniqueIDFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + }, + CountryOrRegionFieldName: "Philippines", + StateFieldName: "Calabarzon", + CityFieldName: "Tagaytay", + PostalCodeFieldName: "4120", + StreetFieldName: "Prime Residences CityLand 1852", + GeoLocFieldName: map[string]any{ + "latitude": 14.1153, + "longitude": 120.962, + }, + DispNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + } + + expectedData := map[string]any{ + "MyAddress": map[string]any{ + AddressFieldName: map[string]any{ + "city": "Tagaytay", + "countryOrRegion": "Philippines", + "postalCode": "4120", + "state": "Calabarzon", + "street": "Prime Residences CityLand 1852", + }, + CoordinatesFieldName: map[string]any{ + "latitude": "14.1153", + "longitude": "120.962", + }, + DisplayNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + LocationURIFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + UniqueIDFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + }, + } + + origFs := models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs := retrieveFieldData(origFs) + fsAdditionalData := fs.GetAdditionalData() + assert.Equal(t, expectedData, fsAdditionalData) +} + type ListsAPIIntgSuite struct { tester.Suite its intgTesterSetup @@ -55,3 +536,322 @@ func (suite *ListsAPIIntgSuite) TestLists_PostDrive() { _, err = acl.PostDrive(ctx, siteID, driveName) require.ErrorIs(t, err, graph.ErrItemAlreadyExistsConflict, clues.ToCore(err)) } + +func (suite *ListsAPIIntgSuite) TestLists_GetListByID() { + var ( + listID = "fake-list-id" + listName = "fake-list-name" + listTemplate = "genericList" + siteID = suite.its.site.id + textColumnDefID = "fake-text-column-id" + textColumnDefName = "itemName" + numColumnDefID = "fake-num-column-id" + numColumnDefName = "itemSize" + colLinkID = "fake-collink-id" + cTypeID = "fake-ctype-id" + listItemID = "fake-list-item-id" + ) + + tests := []struct { + name string + setupf func() + expect assert.ErrorAssertionFunc + }{ + { + name: "", + setupf: func() { + listInfo := models.NewListInfo() + listInfo.SetTemplate(ptr.To(listTemplate)) + + list := models.NewList() + list.SetId(ptr.To(listID)) + list.SetDisplayName(ptr.To(listName)) + list.SetList(listInfo) + list.SetLastModifiedDateTime(ptr.To(time.Now())) + + txtColumnDef := models.NewColumnDefinition() + txtColumnDef.SetId(&textColumnDefID) + txtColumnDef.SetName(&textColumnDefName) + textColumn := models.NewTextColumn() + txtColumnDef.SetText(textColumn) + columnDefCol := models.NewColumnDefinitionCollectionResponse() + columnDefCol.SetValue([]models.ColumnDefinitionable{txtColumnDef}) + + numColumnDef := models.NewColumnDefinition() + numColumnDef.SetId(&numColumnDefID) + numColumnDef.SetName(&numColumnDefName) + numColumn := models.NewNumberColumn() + numColumnDef.SetNumber(numColumn) + columnDefCol2 := models.NewColumnDefinitionCollectionResponse() + columnDefCol2.SetValue([]models.ColumnDefinitionable{numColumnDef}) + + colLink := models.NewColumnLink() + colLink.SetId(&colLinkID) + colLinkCol := models.NewColumnLinkCollectionResponse() + colLinkCol.SetValue([]models.ColumnLinkable{colLink}) + + cTypes := models.NewContentType() + cTypes.SetId(&cTypeID) + cTypesCol := models.NewContentTypeCollectionResponse() + cTypesCol.SetValue([]models.ContentTypeable{cTypes}) + + listItem := models.NewListItem() + listItem.SetId(&listItemID) + listItemCol := models.NewListItemCollectionResponse() + listItemCol.SetValue([]models.ListItemable{listItem}) + + fields := models.NewFieldValueSet() + fieldsData := map[string]any{ + "itemName": "item1", + "itemSize": 4, + } + fields.SetAdditionalData(fieldsData) + + interceptV1Path( + "sites", siteID, + "lists", listID). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), list)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "columns"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), columnDefCol)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "items"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), listItemCol)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "items", listItemID, + "fields"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), fields)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "contentTypes"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), cTypesCol)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "contentTypes", cTypeID, + "columns"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), columnDefCol2)) + + interceptV1Path( + "sites", siteID, + "lists", listID, + "contentTypes", cTypeID, + "columnLinks"). + Reply(200). + JSON(graphTD.ParseableToMap(suite.T(), colLinkCol)) + }, + expect: assert.NoError, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + defer gock.Off() + test.setupf() + + list, info, err := suite.its.gockAC.Lists().GetListByID(ctx, siteID, listID) + test.expect(t, err) + assert.Equal(t, listID, *list.GetId()) + + items := list.GetItems() + assert.Equal(t, 1, len(items)) + assert.Equal(t, listItemID, *items[0].GetId()) + + expectedItemData := map[string]any{"itemName": ptr.To[string]("item1"), "itemSize": ptr.To[float64](float64(4))} + itemFields := items[0].GetFields() + itemData := itemFields.GetAdditionalData() + assert.Equal(t, expectedItemData, itemData) + + columns := list.GetColumns() + assert.Equal(t, 1, len(columns)) + assert.Equal(t, textColumnDefID, *columns[0].GetId()) + + cTypes := list.GetContentTypes() + assert.Equal(t, 1, len(cTypes)) + assert.Equal(t, cTypeID, *cTypes[0].GetId()) + + colLinks := cTypes[0].GetColumnLinks() + assert.Equal(t, 1, len(colLinks)) + assert.Equal(t, colLinkID, *colLinks[0].GetId()) + + columns = cTypes[0].GetColumns() + assert.Equal(t, 1, len(columns)) + assert.Equal(t, numColumnDefID, *columns[0].GetId()) + + assert.Equal(t, listName, info.List.Name) + assert.Equal(t, int64(1), info.List.ItemCount) + assert.Equal(t, listTemplate, info.List.Template) + assert.NotEmpty(t, info.List.Modified) + assert.NotEmpty(t, info.Modified) + }) + } +} + +func (suite *ListsAPIIntgSuite) TestLists_PostList() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + fieldsData, list := getFieldsDataAndList() + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + newList, err := acl.PostList(ctx, siteID, listName, oldListByteArray) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, listName, ptr.Val(newList.GetDisplayName())) + + _, err = acl.PostList(ctx, siteID, listName, oldListByteArray) + require.Error(t, err) + + newListItems := newList.GetItems() + require.Less(t, 0, len(newListItems)) + + newListItemFields := newListItems[0].GetFields() + require.NotEmpty(t, newListItemFields) + + newListItemsData := newListItemFields.GetAdditionalData() + require.NotEmpty(t, newListItemsData) + + for k, v := range newListItemsData { + assert.Equal(t, fieldsData[k], ptr.Val(v.(*string))) + } + + err = acl.DeleteList(ctx, siteID, ptr.Val(newList.GetId())) + require.NoError(t, err) +} + +func (suite *ListsAPIIntgSuite) TestLists_PostList_invalidTemplate() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + overrideListInfo := models.NewListInfo() + overrideListInfo.SetTemplate(ptr.To(WebTemplateExtensionsListTemplateName)) + + _, list := getFieldsDataAndList() + list.SetList(overrideListInfo) + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + _, err = acl.PostList(ctx, siteID, listName, oldListByteArray) + require.Error(t, err) + assert.Equal(t, ErrCannotCreateWebTemplateExtension.Error(), err.Error()) +} + +func (suite *ListsAPIIntgSuite) TestLists_DeleteList() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + _, list := getFieldsDataAndList() + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + newList, err := acl.PostList(ctx, siteID, listName, oldListByteArray) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, listName, ptr.Val(newList.GetDisplayName())) + + err = acl.DeleteList(ctx, siteID, ptr.Val(newList.GetId())) + require.NoError(t, err) +} + +func getFieldsDataAndList() (map[string]any, *models.List) { + oldListID := "old-list" + listItemID := "list-item1" + textColumnDefID := "list-col1" + textColumnDefName := "itemName" + template := "genericList" + + listInfo := models.NewListInfo() + listInfo.SetTemplate(&template) + + textColumn := models.NewTextColumn() + + txtColumnDef := models.NewColumnDefinition() + txtColumnDef.SetId(&textColumnDefID) + txtColumnDef.SetName(&textColumnDefName) + txtColumnDef.SetText(textColumn) + + fields := models.NewFieldValueSet() + fieldsData := map[string]any{ + textColumnDefName: "item1", + } + fields.SetAdditionalData(fieldsData) + + listItem := models.NewListItem() + listItem.SetId(&listItemID) + listItem.SetFields(fields) + + list := models.NewList() + list.SetId(&oldListID) + list.SetList(listInfo) + list.SetColumns([]models.ColumnDefinitionable{txtColumnDef}) + list.SetItems([]models.ListItemable{listItem}) + + return fieldsData, list +}