sets list item fields based on columns created (#4969)

sets list item fields based on columns created while list creation

#### Does this PR need a docs update or release note?
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)
#4754 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/pkg/services/m365/api/lists.go

@@ -25,6 +25,7 @@ const (
 	AuthorLookupIDColumnName    = "AuthorLookupId"
 	EditorLookupIDColumnName    = "EditorLookupId"
 	AppAuthorLookupIDColumnName = "AppAuthorLookupId"
+	TitleColumnName             = "Title"
 
 	ContentTypeColumnDisplayName = "Content Type"
 
@@ -56,38 +57,12 @@ const (
 	AccessRequestsListTemplate  = "accessRequest"
 )
 
-var addressFieldNames = []string{
-	AddressFieldName,
-	CoordinatesFieldName,
-	DisplayNameFieldName,
-	LocationURIFieldName,
-	UniqueIDFieldName,
-}
-
-var readOnlyAddressFieldNames = []string{
-	CountryOrRegionFieldName,
-	StateFieldName,
-	CityFieldName,
-	PostalCodeFieldName,
-	StreetFieldName,
-	GeoLocFieldName,
-	DispNameFieldName,
-}
-
 var legacyColumns = keys.Set{
 	AttachmentsColumnName:        {},
 	EditColumnName:               {},
 	ContentTypeColumnDisplayName: {},
 }
 
-var readOnlyFieldNames = keys.Set{
-	AttachmentsColumnName: {},
-	EditColumnName:        {},
-	ContentTypeColumnName: {},
-	CreatedColumnName:     {},
-	ModifiedColumnName:    {},
-}
-
 var SkipListTemplates = keys.Set{
 	WebTemplateExtensionsListTemplate: {},
 	DocumentLibraryListTemplate:       {},
@@ -266,7 +241,7 @@ func (c Lists) PostList(
 	}
 
 	// this ensure all columns, contentTypes are set to the newList
-	newList := ToListable(oldList, newListName)
+	newList, columnNames := ToListable(oldList, newListName)
 
 	if newList.GetList() != nil &&
 		SkipListTemplates.HasKey(ptr.Val(newList.GetList().GetTemplate())) {
@@ -287,7 +262,7 @@ func (c Lists) PostList(
 	listItems := make([]models.ListItemable, 0)
 
 	for _, itm := range oldList.GetItems() {
-		temp := CloneListItem(itm)
+		temp := CloneListItem(itm, columnNames)
 		listItems = append(listItems, temp)
 	}
 
@@ -360,7 +335,7 @@ func BytesToListable(bytes []byte) (models.Listable, error) {
 // not attached in this method.
 // ListItems are not included in creation of new list, and have to be restored
 // in separate call.
-func ToListable(orig models.Listable, displayName string) models.Listable {
+func ToListable(orig models.Listable, displayName string) (models.Listable, map[string]any) {
 	newList := models.NewList()
 
 	newList.SetContentTypes(orig.GetContentTypes())
@@ -377,6 +352,7 @@ func ToListable(orig models.Listable, displayName string) models.Listable {
 	newList.SetParentReference(orig.GetParentReference())
 
 	columns := make([]models.ColumnDefinitionable, 0)
+	columnNames := map[string]any{TitleColumnName: nil}
 
 	for _, cd := range orig.GetColumns() {
 		var (
@@ -394,16 +370,17 @@ func ToListable(orig models.Listable, displayName string) models.Listable {
 
 		// Skips columns that cannot be uploaded for models.ColumnDefinitionable:
 		// - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type
-		if readOnly || displayName == "Title" || legacyColumns.HasKey(displayName) {
+		if readOnly || displayName == TitleColumnName || legacyColumns.HasKey(displayName) {
 			continue
 		}
 
 		columns = append(columns, cloneColumnDefinitionable(cd))
+		columnNames[ptr.Val(cd.GetName())] = nil
 	}
 
 	newList.SetColumns(columns)
 
-	return newList
+	return newList, columnNames
 }
 
 // cloneColumnDefinitionable utility function for encapsulating models.ColumnDefinitionable data
@@ -486,11 +463,11 @@ func setColumnType(newColumn *models.ColumnDefinition, orig models.ColumnDefinit
 // CloneListItem creates a new `SharePoint.ListItem` and stores the original item's
 // M365 data into it set fields.
 // - https://learn.microsoft.com/en-us/graph/api/resources/listitem?view=graph-rest-1.0
-func CloneListItem(orig models.ListItemable) models.ListItemable {
+func CloneListItem(orig models.ListItemable, columnNames map[string]any) models.ListItemable {
 	newItem := models.NewListItem()
 
 	// list item data
-	newFieldData := retrieveFieldData(orig.GetFields())
+	newFieldData := retrieveFieldData(orig.GetFields(), columnNames)
 	newItem.SetFields(newFieldData)
 
 	// list item attributes
@@ -523,18 +500,19 @@ func CloneListItem(orig models.ListItemable) models.ListItemable {
 // additionalData map
 // Further documentation on FieldValueSets:
 // - https://learn.microsoft.com/en-us/graph/api/resources/fieldvalueset?view=graph-rest-1.0
-func retrieveFieldData(orig models.FieldValueSetable) models.FieldValueSetable {
+func retrieveFieldData(orig models.FieldValueSetable, columnNames map[string]any) models.FieldValueSetable {
 	fields := models.NewFieldValueSet()
-	additionalData := filterAdditionalData(orig)
-
-	retainPrimaryAddressField(additionalData)
 
+	additionalData := setAdditionalDataByColumnNames(orig, columnNames)
 	fields.SetAdditionalData(additionalData)
 
 	return fields
 }
 
-func filterAdditionalData(orig models.FieldValueSetable) map[string]any {
+func setAdditionalDataByColumnNames(
+	orig models.FieldValueSetable,
+	columnNames map[string]any,
+) map[string]any {
 	if orig == nil {
 		return make(map[string]any)
 	}
@@ -542,55 +520,15 @@ func filterAdditionalData(orig models.FieldValueSetable) map[string]any {
 	fieldData := orig.GetAdditionalData()
 	filteredData := make(map[string]any)
 
-	for key, value := range fieldData {
+	for colName := range columnNames {
-		if shouldFilterField(key, value) {
+		if _, ok := fieldData[colName]; ok {
-			continue
+			filteredData[colName] = fieldData[colName]
 		}
-
-		filteredData[key] = value
 	}
 
 	return filteredData
 }
 
-func shouldFilterField(key string, value any) bool {
-	return readOnlyFieldNames.HasKey(key) ||
-		strings.HasPrefix(key, ReadOnlyOrHiddenFieldNamePrefix) ||
-		strings.HasPrefix(key, DescoratorFieldNamePrefix) ||
-		strings.Contains(key, LinkTitleFieldNamePart) ||
-		strings.Contains(key, ChildCountFieldNamePart) ||
-		strings.Contains(key, LookupIDFieldNamePart)
-}
-
-func retainPrimaryAddressField(additionalData map[string]any) {
-	if !hasAddressFields(additionalData) {
-		return
-	}
-
-	for _, k := range readOnlyAddressFieldNames {
-		delete(additionalData, k)
-	}
-}
-
-func hasAddressFields(additionalData map[string]any) bool {
-	if !keys.HasKeys(additionalData, readOnlyAddressFieldNames...) {
-		return false
-	}
-
-	for _, value := range additionalData {
-		nestedFields, ok := value.(map[string]any)
-		if !ok || keys.HasKeys(nestedFields, GeoLocFieldName) {
-			continue
-		}
-
-		if keys.HasKeys(nestedFields, addressFieldNames...) {
-			return true
-		}
-	}
-
-	return false
-}
-
 func (c Lists) getListItemFields(
 	ctx context.Context,
 	siteID, listID, itemID string,

src/pkg/services/m365/api/lists_test.go

@@ -328,9 +328,10 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 	roCd.SetReadOnly(ptr.To(true))
 
 	tests := []struct {
-		name    string
+		name             string
-		getList func() *models.List
+		getList          func() *models.List
-		length  int
+		length           int
+		expectedColNames map[string]any
 	}{
 		{
 			name: "all legacy columns",
@@ -343,7 +344,8 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 				})
 				return lst
 			},
-			length: 0,
+			length:           0,
+			expectedColNames: map[string]any{TitleColumnName: nil},
 		},
 		{
 			name: "title and legacy columns",
@@ -357,7 +359,8 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 				})
 				return lst
 			},
-			length: 0,
+			length:           0,
+			expectedColNames: map[string]any{TitleColumnName: nil},
 		},
 		{
 			name: "readonly and legacy columns",
@@ -371,7 +374,8 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 				})
 				return lst
 			},
-			length: 0,
+			length:           0,
+			expectedColNames: map[string]any{TitleColumnName: nil},
 		},
 		{
 			name: "legacy and a text column",
@@ -386,6 +390,10 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 				return lst
 			},
 			length: 1,
+			expectedColNames: map[string]any{
+				TitleColumnName: nil,
+				textColumnName:  nil,
+			},
 		},
 	}
 
@@ -393,8 +401,9 @@ func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() {
 		suite.Run(test.name, func() {
 			t := suite.T()
 
-			clonedList := ToListable(test.getList(), listName)
+			clonedList, colNames := ToListable(test.getList(), listName)
 			require.NotEmpty(t, clonedList)
+			assert.Equal(t, test.expectedColNames, colNames)
 
 			cols := clonedList.GetColumns()
 			assert.Len(t, cols, test.length)
@@ -422,7 +431,9 @@ func (suite *ListsUnitSuite) TestFieldValueSetable() {
 	origFs := models.NewFieldValueSet()
 	origFs.SetAdditionalData(additionalData)
 
-	fs := retrieveFieldData(origFs)
+	colNames := map[string]any{}
+
+	fs := retrieveFieldData(origFs, colNames)
 	fsAdditionalData := fs.GetAdditionalData()
 	assert.Empty(t, fsAdditionalData)
 
@@ -430,7 +441,9 @@ func (suite *ListsUnitSuite) TestFieldValueSetable() {
 	origFs = models.NewFieldValueSet()
 	origFs.SetAdditionalData(additionalData)
 
-	fs = retrieveFieldData(origFs)
+	colNames["itemName"] = struct{}{}
+
+	fs = retrieveFieldData(origFs, colNames)
 	fsAdditionalData = fs.GetAdditionalData()
 	assert.NotEmpty(t, fsAdditionalData)
 
@@ -493,7 +506,11 @@ func (suite *ListsUnitSuite) TestFieldValueSetable_Location() {
 	origFs := models.NewFieldValueSet()
 	origFs.SetAdditionalData(additionalData)
 
-	fs := retrieveFieldData(origFs)
+	colNames := map[string]any{
+		"MyAddress": nil,
+	}
+
+	fs := retrieveFieldData(origFs, colNames)
 	fsAdditionalData := fs.GetAdditionalData()
 	assert.Equal(t, expectedData, fsAdditionalData)
 }