fix: adapt to the valid_data change in milvus-proto (#52203)

issue: https://github.com/milvus-io/milvus/issues/51162

---------

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>

.golangci.yml

@@ -134,6 +134,18 @@ linters:
   exclusions:
     generated: lax
     rules:
+      - linters:
+          - gocritic
+        path: .+_test\.go
+        text: do not access protobuf valid_data directly
+      - linters:
+          - gocritic
+        path: (^|/)util/typeutil/field_data\.go$
+        text: do not access protobuf valid_data directly
+      - linters:
+          - gocritic
+        path: (^|/)util/fastpb/fielddata\.go$
+        text: do not access protobuf valid_data directly
       - linters:
           - forbidigo
         path: .+_test\.go

client/.golangci.yml

@@ -104,6 +104,14 @@ linters:
   exclusions:
     generated: lax
     rules:
+      - linters:
+          - gocritic
+        path: .+_test\.go
+        text: do not access protobuf valid_data directly
+      - linters:
+          - gocritic
+        path: ^column/generic_base\.go$
+        text: do not access protobuf valid_data directly
       - linters:
           - forbidigo
         path: .+_test\.go

client/column/array.go

@@ -32,7 +32,6 @@ func (c *columnArrayBase[T]) FieldData() *schemapb.FieldData {
 	fd := &schemapb.FieldData{
 		Type:      schemapb.DataType_Array,
 		FieldName: c.name,
-		ValidData: c.validData,
 	}
 
 	data := make([]*schemapb.ScalarField, 0, c.Len())
@@ -50,6 +49,9 @@ func (c *columnArrayBase[T]) FieldData() *schemapb.FieldData {
 			},
 		},
 	}
+	if c.nullable {
+		setFieldDataValidData(fd, c.validData)
+	}
 	return fd
 }
 

client/column/array_test.go

@@ -301,6 +301,16 @@ func (s *ArraySuite) TestBasic() {
 	})
 }
 
+func (s *ArraySuite) TestNullableFieldDataValidity() {
+	column := NewColumnInt64Array("array", [][]int64{{1, 2}})
+	column.SetNullable(true)
+	s.Require().NoError(column.AppendNull())
+
+	fd := column.FieldData()
+	s.Nil(fd.GetValidData())
+	s.Equal([]bool{true, false}, fd.GetScalars().GetValidData())
+}
+
 func TestArrays(t *testing.T) {
 	suite.Run(t, new(ArraySuite))
 }

client/column/columns.go

@@ -426,7 +426,10 @@ func int32ToType[T ~int8 | int16](data []int32) []T {
 // FieldDataColumn converts schemapb.FieldData to Column, used int search result conversion logic
 // begin, end specifies the start and end positions
 func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) {
-	validData := fd.GetValidData()
+	if !validateAndNormalizeFieldDataValidData(fd) {
+		return nil, fmt.Errorf("field %q has different legacy and field-specific valid_data", fd.GetFieldName())
+	}
+	validData := getFieldDataValidData(fd)
 
 	switch fd.GetType() {
 	case schemapb.DataType_Bool:

client/column/columns_test.go

@@ -99,6 +99,71 @@ func TestIDColumns(t *testing.T) {
 	})
 }
 
+func TestFieldDataColumnValidDataSources(t *testing.T) {
+	validData := []bool{true, false}
+	makeField := func(legacy, current []bool) *schemapb.FieldData {
+		return &schemapb.FieldData{
+			Type:      schemapb.DataType_Int64,
+			FieldName: "value",
+			ValidData: legacy,
+			Field: &schemapb.FieldData_Scalars{
+				Scalars: &schemapb.ScalarField{
+					ValidData: current,
+					Data: &schemapb.ScalarField_LongData{
+						LongData: &schemapb.LongArray{Data: []int64{1, 0}},
+					},
+				},
+			},
+		}
+	}
+
+	for _, test := range []struct {
+		name    string
+		legacy  []bool
+		current []bool
+		wantErr bool
+	}{
+		{name: "legacy fallback", legacy: validData},
+		{name: "field-specific", current: validData},
+		{name: "matching dual sources", legacy: validData, current: validData},
+		{name: "mismatched dual sources", legacy: validData, current: []bool{false, true}, wantErr: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			field := makeField(test.legacy, test.current)
+			col, err := FieldDataColumn(field, 0, -1)
+			if test.wantErr {
+				assert.Error(t, err)
+				assert.Nil(t, col)
+				return
+			}
+			assert.NoError(t, err)
+			assert.NotNil(t, col)
+			assert.Nil(t, field.GetValidData())
+			assert.Equal(t, validData, field.GetScalars().GetValidData())
+		})
+	}
+}
+
+func TestValidateAndNormalizeFieldDataValidDataRejectsNestedMismatch(t *testing.T) {
+	legacy := []bool{true, false}
+	current := []bool{false, true}
+	subField := &schemapb.FieldData{
+		ValidData: legacy,
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{ValidData: current},
+		},
+	}
+	field := &schemapb.FieldData{
+		Field: &schemapb.FieldData_StructArrays{
+			StructArrays: &schemapb.StructArrayField{Fields: []*schemapb.FieldData{subField}},
+		},
+	}
+
+	assert.False(t, validateAndNormalizeFieldDataValidData(field))
+	assert.Equal(t, legacy, subField.GetValidData())
+	assert.Equal(t, current, subField.GetScalars().GetValidData())
+}
+
 func TestGetIntData(t *testing.T) {
 	type testCase struct {
 		tag      string

client/column/generic_base.go

@@ -31,6 +31,88 @@ type GColumn[T any] interface {
 	AppendValue(v T)
 }
 
+func getFieldDataValidData(fd *schemapb.FieldData) []bool {
+	if legacy := fd.GetValidData(); len(legacy) > 0 {
+		return legacy
+	}
+	var current []bool
+	if scalars := fd.GetScalars(); scalars != nil {
+		current = scalars.GetValidData()
+	} else {
+		current = fd.GetVectors().GetValidData()
+	}
+	return current
+}
+
+func setFieldDataValidData(fd *schemapb.FieldData, validData []bool) {
+	if fd == nil {
+		return
+	}
+
+	if scalars := fd.GetScalars(); scalars != nil {
+		scalars.ValidData = validData
+	} else if vectors := fd.GetVectors(); vectors != nil {
+		vectors.ValidData = validData
+	} else {
+		return
+	}
+
+	fd.ValidData = nil
+}
+
+func validateAndNormalizeFieldDataValidData(fd *schemapb.FieldData) bool {
+	if !fieldDataValidDataConsistent(fd) {
+		return false
+	}
+	normalizeFieldDataValidData(fd)
+	return true
+}
+
+func fieldDataValidDataConsistent(fd *schemapb.FieldData) bool {
+	if fd == nil {
+		return true
+	}
+
+	legacy := fd.GetValidData()
+	var current []bool
+	if scalars := fd.GetScalars(); scalars != nil {
+		current = scalars.GetValidData()
+	} else {
+		current = fd.GetVectors().GetValidData()
+	}
+	if len(legacy) > 0 && len(current) > 0 && !slices.Equal(legacy, current) {
+		return false
+	}
+
+	for _, subField := range fd.GetStructArrays().GetFields() {
+		if !fieldDataValidDataConsistent(subField) {
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeFieldDataValidData(fd *schemapb.FieldData) {
+	if fd == nil {
+		return
+	}
+	switch fd.Field.(type) {
+	case *schemapb.FieldData_Scalars, *schemapb.FieldData_Vectors:
+		if validData := getFieldDataValidData(fd); len(validData) > 0 {
+			setFieldDataValidData(fd, validData)
+		} else {
+			fd.ValidData = nil
+		}
+	case *schemapb.FieldData_StructArrays:
+		fd.ValidData = nil
+		for _, subField := range fd.GetStructArrays().GetFields() {
+			normalizeFieldDataValidData(subField)
+		}
+	default:
+		fd.ValidData = nil
+	}
+}
+
 var _ Column = (*genericColumnBase[any])(nil)
 
 // genericColumnBase implements `Column` interface
@@ -153,7 +235,7 @@ func (c *genericColumnBase[T]) FieldData() *schemapb.FieldData {
 	fd.FieldName = c.name
 	fd.Type = schemapb.DataType(c.fieldType)
 	if c.nullable {
-		fd.ValidData = c.validData
+		setFieldDataValidData(fd, c.validData)
 	}
 	return fd
 }

client/column/nullable_test.go

@@ -57,7 +57,8 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.Equal(sparseData, column.Data())
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
+		s.Nil(fd.GetValidData())
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnBool)
@@ -93,7 +94,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnInt8)
@@ -129,7 +130,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnInt16)
@@ -165,7 +166,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnInt32)
@@ -201,7 +202,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnInt64)
@@ -237,7 +238,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnFloat)
@@ -273,7 +274,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnDouble)
@@ -320,7 +321,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnTimestampTzIsoString)
@@ -356,7 +357,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 
 		fd := column.FieldData()
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		result, err := FieldDataColumn(fd, 0, -1)
 		s.NoError(err)
 		parsed, ok := result.(*ColumnTimestampTzIsoString)
@@ -386,7 +387,7 @@ func (s *NullableScalarSuite) TestBasic() {
 		s.NoError(err)
 		fd := column.FieldData()
 		s.EqualValues(entity.FieldTypeText, fd.GetType())
-		s.Equal(validData, fd.GetValidData())
+		s.Equal(validData, getFieldDataValidData(fd))
 		s.Equal(sparseData, fd.GetScalars().GetStringData().GetData())
 
 		result, err := FieldDataColumn(fd, 0, -1)

client/column/struct_test.go

@@ -109,7 +109,8 @@ func (s *StructArraySuite) TestNullableScalarRows() {
 	s.False(isNull, "an empty struct array is distinct from null")
 
 	for _, fieldData := range column.FieldData().GetStructArrays().GetFields() {
-		s.Equal([]bool{true, false, true}, fieldData.GetValidData())
+		s.Equal([]bool{true, false, true}, getFieldDataValidData(fieldData))
+		s.Nil(fieldData.GetValidData())
 		s.Equal(2, len(fieldData.GetScalars().GetArrayData().GetData()))
 	}
 }
@@ -171,7 +172,8 @@ func (s *StructArraySuite) TestNullableCompactSlice() {
 	s.Require().NoError(err)
 	s.NotNil(value)
 	for _, fieldData := range sliced.FieldData().GetStructArrays().GetFields() {
-		s.Equal([]bool{false, true}, fieldData.GetValidData())
+		s.Equal([]bool{false, true}, getFieldDataValidData(fieldData))
+		s.Nil(fieldData.GetValidData())
 	}
 }
 
@@ -262,7 +264,8 @@ func (s *StructArraySuite) TestNullableVectorRows() {
 	fields := column.FieldData().GetStructArrays().GetFields()
 	s.Require().Len(fields, 2)
 	for _, fieldData := range fields {
-		s.Equal([]bool{true, false, true}, fieldData.GetValidData())
+		s.Equal([]bool{true, false, true}, getFieldDataValidData(fieldData))
+		s.Nil(fieldData.GetValidData())
 	}
 	s.Len(fields[0].GetScalars().GetArrayData().GetData(), 2)
 	s.Len(fields[1].GetVectors().GetVectorArray().GetData(), 2)
@@ -386,8 +389,8 @@ func (s *StructArraySuite) TestParseNullableStructArray() {
 	intSub := &schemapb.FieldData{
 		Type:      schemapb.DataType_Array,
 		FieldName: "id",
-		ValidData: validData,
 		Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
+			ValidData: validData,
 			Data: &schemapb.ScalarField_ArrayData{ArrayData: &schemapb.ArrayArray{
 				ElementType: schemapb.DataType_Int64,
 				Data: []*schemapb.ScalarField{
@@ -401,8 +404,8 @@ func (s *StructArraySuite) TestParseNullableStructArray() {
 	strSub := &schemapb.FieldData{
 		Type:      schemapb.DataType_Array,
 		FieldName: "tag",
-		ValidData: validData,
 		Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
+			ValidData: validData,
 			Data: &schemapb.ScalarField_ArrayData{ArrayData: &schemapb.ArrayArray{
 				ElementType: schemapb.DataType_VarChar,
 				Data: []*schemapb.ScalarField{
@@ -488,8 +491,9 @@ func (s *StructArraySuite) TestParseNullableStructArrayEmptySlice() {
 	s.Zero(parsed.Len())
 	s.Require().NoError(parsed.ValidateNullable())
 	for _, fieldData := range parsed.FieldData().GetStructArrays().GetFields() {
-		s.NotNil(fieldData.ValidData)
-		s.Empty(fieldData.GetValidData())
+		s.NotNil(getFieldDataValidData(fieldData))
+		s.Empty(getFieldDataValidData(fieldData))
+		s.Nil(fieldData.GetValidData())
 	}
 }
 
@@ -500,8 +504,9 @@ func (s *StructArraySuite) TestParseNullableStructArrayEmptyRows() {
 	})
 	source.SetNullable(true)
 	for _, fieldData := range source.FieldData().GetStructArrays().GetFields() {
-		s.NotNil(fieldData.ValidData)
-		s.Empty(fieldData.GetValidData())
+		s.NotNil(getFieldDataValidData(fieldData))
+		s.Empty(getFieldDataValidData(fieldData))
+		s.Nil(fieldData.GetValidData())
 	}
 
 	parsed, err := FieldDataColumn(source.FieldData(), 0, -1)
@@ -558,8 +563,8 @@ func (s *StructArraySuite) TestParseNullableStructArrayRejectsMismatchedMasks()
 		return &schemapb.FieldData{
 			Type:      schemapb.DataType_Array,
 			FieldName: name,
-			ValidData: validData,
 			Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
+				ValidData: validData,
 				Data: &schemapb.ScalarField_ArrayData{ArrayData: &schemapb.ArrayArray{
 					ElementType: schemapb.DataType_Int64,
 					Data: []*schemapb.ScalarField{

client/column/vector_array.go

@@ -241,7 +241,7 @@ func (c *columnVectorArrayBase[T]) FieldData() *schemapb.FieldData {
 		},
 	}
 	if c.nullable {
-		fd.ValidData = c.validData
+		setFieldDataValidData(fd, c.validData)
 	}
 	return fd
 }

client/column/vector_array_test.go

@@ -104,7 +104,8 @@ func (s *VectorArraySuite) TestFloatVectorArrayBasic() {
 	s.EqualValues(dim, va.GetDim())
 	s.Equal(schemapb.DataType_FloatVector, va.GetElementType())
 	s.Equal(4, len(va.GetData()))
-	s.Equal([]bool{true, true, false, true, true}, fd.GetValidData())
+	s.Equal([]bool{true, true, false, true, true}, getFieldDataValidData(fd))
+	s.Nil(fd.GetValidData())
 }
 
 func (s *VectorArraySuite) TestFloat16VectorArrayBasic() {
@@ -231,9 +232,9 @@ func (s *VectorArraySuite) TestParseNullableVectorArrayData() {
 	fd := &schemapb.FieldData{
 		Type:      schemapb.DataType_ArrayOfVector,
 		FieldName: "emb",
-		ValidData: []bool{true, false, true},
 		Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-			Dim: int64(dim),
+			Dim:       int64(dim),
+			ValidData: []bool{true, false, true},
 			Data: &schemapb.VectorField_VectorArray{VectorArray: &schemapb.VectorArray{
 				Dim:         int64(dim),
 				ElementType: schemapb.DataType_FloatVector,
@@ -440,10 +441,10 @@ func (s *VectorArraySuite) TestParseAllNullVectorArrayUsesOuterDim() {
 	fd := &schemapb.FieldData{
 		Type:      schemapb.DataType_ArrayOfVector,
 		FieldName: "emb",
-		ValidData: []bool{false, false},
 		Field: &schemapb.FieldData_Vectors{
 			Vectors: &schemapb.VectorField{
-				Dim: 2,
+				Dim:       2,
+				ValidData: []bool{false, false},
 				Data: &schemapb.VectorField_VectorArray{
 					VectorArray: &schemapb.VectorArray{
 						ElementType: schemapb.DataType_FloatVector,
@@ -501,10 +502,10 @@ func (s *VectorArraySuite) TestCompactSliceDoesNotMutateSource() {
 	fd := &schemapb.FieldData{
 		Type:      schemapb.DataType_ArrayOfVector,
 		FieldName: "emb",
-		ValidData: []bool{false, true, true},
 		Field: &schemapb.FieldData_Vectors{
 			Vectors: &schemapb.VectorField{
-				Dim: 2,
+				Dim:       2,
+				ValidData: []bool{false, true, true},
 				Data: &schemapb.VectorField_VectorArray{
 					VectorArray: &schemapb.VectorArray{
 						Dim:         2,

client/milvusclient/read_test.go

@@ -210,7 +210,8 @@ func (s *ReadSuite) TestParseEmptyNullableStructArrayFromWire() {
 	decoded := &schemapb.FieldData{}
 	s.Require().NoError(proto.Unmarshal(wire, decoded))
 	for _, subField := range decoded.GetStructArrays().GetFields() {
-		s.Nil(subField.ValidData)
+		s.Empty(subField.GetScalars().GetValidData())
+		s.Empty(subField.GetVectors().GetValidData())
 	}
 
 	columns, err := s.client.parseSearchResult(schema, []string{"clips"}, []*schemapb.FieldData{decoded}, 0, 0, 0)
@@ -272,7 +273,7 @@ func (s *ReadSuite) TestHandleSearchResultSlicesCompactNullableFields() {
 		WithField(entity.NewField().WithName("ID").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
 		WithField(entity.NewField().WithName("age").WithDataType(entity.FieldTypeInt64).WithNullable(true))
 	age := s.getInt64FieldData("age", []int64{10, 40})
-	age.ValidData = []bool{true, false, false, true}
+	age.GetScalars().ValidData = []bool{true, false, false, true}
 	resp := &milvuspb.SearchResults{
 		Results: &schemapb.SearchResultData{
 			NumQueries: 2,
@@ -315,7 +316,7 @@ func (s *ReadSuite) TestHandleSearchResultPreservesNullableGeometryRows() {
 		WithField(entity.NewField().WithName("ID").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
 		WithField(entity.NewField().WithName("location").WithDataType(entity.FieldTypeGeometry).WithNullable(true))
 	location := s.getGeometryWktFieldData("location", []string{"POINT (1 2)"})
-	location.ValidData = []bool{false, true}
+	location.GetScalars().ValidData = []bool{false, true}
 	resp := &milvuspb.SearchResults{
 		Results: &schemapb.SearchResultData{
 			NumQueries: 1,

client/milvusclient/write_option_test.go

@@ -44,7 +44,7 @@ func (s *ColumnBasedDataOptionSuite) NullableCompatible() {
 	s.Require().Len(req.GetFieldsData(), 1)
 	fd := req.GetFieldsData()[0]
 	s.ElementsMatch([]int64{1, 2, 3}, fd.GetScalars().GetLongData())
-	s.ElementsMatch([]bool{true, true, true}, fd.GetValidData())
+	s.ElementsMatch([]bool{true, true, true}, fd.GetScalars().GetValidData())
 }
 
 func (s *ColumnBasedDataOptionSuite) TestWithStructArrayColumn() {
@@ -159,7 +159,11 @@ func (s *ColumnBasedDataOptionSuite) TestWithNullableStructArrayColumn() {
 	subs := clipsFD.GetStructArrays().GetFields()
 	s.Require().Len(subs, 2)
 	for _, sub := range subs {
-		s.Equal([]bool{true, false, true}, sub.GetValidData())
+		if sub.GetScalars() != nil {
+			s.Equal([]bool{true, false, true}, sub.GetScalars().GetValidData())
+		} else {
+			s.Equal([]bool{true, false, true}, sub.GetVectors().GetValidData())
+		}
 	}
 	s.Len(subs[0].GetScalars().GetArrayData().GetData(), 2)
 	s.Len(subs[1].GetVectors().GetVectorArray().GetData(), 2)

client/ruleguard/rules.go

@@ -407,3 +407,29 @@ func badlock(m dsl.Matcher) {
 	m.Match(`$mu.Lock(); defer $mu.RUnlock()`).Report(`maybe $mu.RLock() was intended?`)
 	m.Match(`$mu.RLock(); defer $mu.Unlock()`).Report(`maybe $mu.Lock() was intended?`)
 }
+
+// fieldDataValidDataAccess prevents client code from depending on protobuf
+// validity storage details. The compatibility implementation in
+// column/generic_base.go is the sole exception because the client module is
+// intentionally independent from the server's pkg/v3/typeutil module.
+func fieldDataValidDataAccess(m dsl.Matcher) {
+	m.Import("github.com/milvus-io/milvus-proto/go-api/v3/schemapb")
+	isFieldDataType := func(v dsl.Var) bool {
+		return v.Type.Is("schemapb.FieldData") || v.Type.Is("*schemapb.FieldData") ||
+			v.Type.Is("schemapb.ScalarField") || v.Type.Is("*schemapb.ScalarField") ||
+			v.Type.Is("schemapb.VectorField") || v.Type.Is("*schemapb.VectorField")
+	}
+	// Match selectors as expressions so reads, writes, getter calls, and getter
+	// method values are all rejected for both pointer and value receivers.
+	m.Match("$x.ValidData", "$x.GetValidData").
+		Where(isFieldDataType(m["x"])).
+		Report("do not access protobuf valid_data directly; use the validity helpers in column/generic_base.go")
+
+	// A keyed composite literal bypasses selector matching, so reject setting
+	// ValidData while constructing any of the three protobuf message types too.
+	m.Match(`$x{$*_, ValidData: $_, $*_}`).
+		Where(m["x"].Type.Is("schemapb.FieldData") ||
+			m["x"].Type.Is("schemapb.ScalarField") ||
+			m["x"].Type.Is("schemapb.VectorField")).
+		Report("do not access protobuf valid_data directly; use the validity helpers in column/generic_base.go")
+}

internal/agg/aggregate_util.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/pkg/v3/util/merr"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 func NewFieldAccessor(fieldType schemapb.DataType) (FieldAccessor, error) {
@@ -68,7 +69,7 @@ func (i32Field *Int32FieldAccessor) Hash(idx int) uint64 {
 
 func (i32Field *Int32FieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	i32Field.vals = fieldData.GetScalars().GetIntData().GetData()
-	i32Field.validData = fieldData.GetValidData()
+	i32Field.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (i32Field *Int32FieldAccessor) RowCount() int {
@@ -113,7 +114,7 @@ func (i64Field *Int64FieldAccessor) Hash(idx int) uint64 {
 
 func (i64Field *Int64FieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	i64Field.vals = fieldData.GetScalars().GetLongData().GetData()
-	i64Field.validData = fieldData.GetValidData()
+	i64Field.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (i64Field *Int64FieldAccessor) RowCount() int {
@@ -158,7 +159,7 @@ func (tzField *TimestamptzFieldAccessor) Hash(idx int) uint64 {
 
 func (tzField *TimestamptzFieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	tzField.vals = fieldData.GetScalars().GetTimestamptzData().GetData()
-	tzField.validData = fieldData.GetValidData()
+	tzField.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (tzField *TimestamptzFieldAccessor) RowCount() int {
@@ -208,7 +209,7 @@ func (boolField *BoolFieldAccessor) Hash(idx int) uint64 {
 
 func (boolField *BoolFieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	boolField.vals = fieldData.GetScalars().GetBoolData().GetData()
-	boolField.validData = fieldData.GetValidData()
+	boolField.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (boolField *BoolFieldAccessor) RowCount() int {
@@ -254,7 +255,7 @@ func (f32FieldAccessor *Float32FieldAccessor) Hash(idx int) uint64 {
 
 func (f32FieldAccessor *Float32FieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	f32FieldAccessor.vals = fieldData.GetScalars().GetFloatData().GetData()
-	f32FieldAccessor.validData = fieldData.GetValidData()
+	f32FieldAccessor.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (f32FieldAccessor *Float32FieldAccessor) RowCount() int {
@@ -300,7 +301,7 @@ func (f64Field *Float64FieldAccessor) Hash(idx int) uint64 {
 
 func (f64Field *Float64FieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	f64Field.vals = fieldData.GetScalars().GetDoubleData().GetData()
-	f64Field.validData = fieldData.GetValidData()
+	f64Field.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (f64Field *Float64FieldAccessor) RowCount() int {
@@ -345,7 +346,7 @@ func (stringField *StringFieldAccessor) Hash(idx int) uint64 {
 
 func (stringField *StringFieldAccessor) SetVals(fieldData *schemapb.FieldData) {
 	stringField.vals = fieldData.GetScalars().GetStringData().GetData()
-	stringField.validData = fieldData.GetValidData()
+	stringField.validData = typeutil.GetFieldDataValidData(fieldData)
 }
 
 func (stringField *StringFieldAccessor) RowCount() int {
@@ -391,7 +392,7 @@ func AssembleSingleRow(colCount int, row *Row, fieldDatas []*schemapb.FieldData)
 func AssembleSingleValue(fv *FieldValue, fieldData *schemapb.FieldData) error {
 	isNull := fv.IsNull()
 	// Append validity data (true = valid, false = null)
-	fieldData.ValidData = append(fieldData.ValidData, !isNull)
+	typeutil.SetFieldDataValidData(fieldData, append(typeutil.GetFieldDataValidData(fieldData), !isNull))
 
 	// For null values, append zero/default values to maintain array alignment
 	if isNull {

internal/core/src/common/Utils.h

@@ -46,6 +46,40 @@ namespace milvus {
 #define VEC_FIELD_DATA(data_array, type) \
     (data_array->vectors().type##_vector().data())
 
+inline const google::protobuf::RepeatedField<bool>&
+GetFieldDataRowValidData(const DataArray& field_data) {
+    // New payloads normally store validity in the field-specific ScalarField
+    // or VectorField. FieldData.valid_data is kept as a fallback for legacy
+    // payloads that do not contain field-specific validity.
+    if (field_data.has_scalars() &&
+        field_data.scalars().valid_data_size() > 0) {
+        return field_data.scalars().valid_data();
+    }
+    if (field_data.has_vectors() &&
+        field_data.vectors().valid_data_size() > 0) {
+        return field_data.vectors().valid_data();
+    }
+    return field_data.valid_data();
+}
+
+inline google::protobuf::RepeatedField<bool>*
+MutableFieldDataRowValidData(DataArray* field_data) {
+    field_data->clear_valid_data();
+    if (field_data->has_scalars()) {
+        return field_data->mutable_scalars()->mutable_valid_data();
+    }
+    if (field_data->has_vectors()) {
+        return field_data->mutable_vectors()->mutable_valid_data();
+    }
+    auto data_type = static_cast<DataType>(field_data->type());
+    AssertInfo(data_type != DataType::NONE,
+               "cannot set valid_data without FieldData type");
+    if (IsVectorDataType(data_type)) {
+        return field_data->mutable_vectors()->mutable_valid_data();
+    }
+    return field_data->mutable_scalars()->mutable_valid_data();
+}
+
 using CheckDataValid = std::function<bool(size_t)>;
 using SparseValueType = typename knowhere::sparse_u32_f32::ValueType;
 

internal/core/src/common/UtilsTest.cpp

@@ -44,6 +44,51 @@ TEST(Util_Common, CheckPlusOverflowKeepsSystemClassification) {
     }
 }
 
+TEST(Util_Common, FieldDataRowValidData) {
+    milvus::DataArray field_data;
+    field_data.add_valid_data(true);
+    field_data.add_valid_data(false);
+
+    EXPECT_EQ(milvus::GetFieldDataRowValidData(field_data).size(), 2);
+    EXPECT_TRUE(milvus::GetFieldDataRowValidData(field_data)[0]);
+    EXPECT_FALSE(milvus::GetFieldDataRowValidData(field_data)[1]);
+
+    field_data.clear_valid_data();
+    auto* scalar_valid_data =
+        field_data.mutable_scalars()->mutable_valid_data();
+    scalar_valid_data->Add(false);
+    scalar_valid_data->Add(true);
+
+    const auto& current = milvus::GetFieldDataRowValidData(field_data);
+    EXPECT_EQ(current.size(), 2);
+    EXPECT_FALSE(current[0]);
+    EXPECT_TRUE(current[1]);
+}
+
+TEST(Util_Common, MutableFieldDataRowValidData) {
+    milvus::DataArray scalar_field;
+    scalar_field.add_valid_data(true);
+    scalar_field.set_type(milvus::proto::schema::DataType::Int64);
+
+    auto* scalar_valid_data =
+        milvus::MutableFieldDataRowValidData(&scalar_field);
+    scalar_valid_data->Add(false);
+    EXPECT_TRUE(scalar_field.valid_data().empty());
+    ASSERT_EQ(scalar_field.scalars().valid_data_size(), 1);
+    EXPECT_FALSE(scalar_field.scalars().valid_data(0));
+
+    milvus::DataArray vector_field;
+    vector_field.add_valid_data(false);
+    vector_field.set_type(milvus::proto::schema::DataType::FloatVector);
+
+    auto* vector_valid_data =
+        milvus::MutableFieldDataRowValidData(&vector_field);
+    vector_valid_data->Add(true);
+    EXPECT_TRUE(vector_field.valid_data().empty());
+    ASSERT_EQ(vector_field.vectors().valid_data_size(), 1);
+    EXPECT_TRUE(vector_field.vectors().valid_data(0));
+}
+
 TEST(SimilarityCorelation, Naive) {
     ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::IP));
     ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::COSINE));

internal/core/src/exec/expression/GISConjunctExpr.cpp

@@ -19,6 +19,7 @@
 #include "common/GeometryCache.h"
 #include "common/OpContext.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "exec/expression/GISFunctionFilterExpr.h"
 #include "geos_c.h"
 #include "index/Index.h"
@@ -445,7 +446,7 @@ PhyGISRefineConjunctExpr::Eval(EvalCtx& context, VectorPtr& result) {
             auto geometry_array =
                 static_cast<const milvus::proto::schema::GeometryArray*>(
                     &data_array->scalars().geometry_data());
-            const auto& vd = data_array->valid_data();
+            const auto& vd = GetFieldDataRowValidData(*data_array);
             GEOSContextHandle_t local_ctx = GetThreadLocalGEOSContext();
             for (size_t k = 0; k < hit_abs.size(); ++k) {
                 if (!vd.empty() && !vd[k]) {

internal/core/src/exec/expression/GISFunctionFilterExpr.cpp

@@ -27,6 +27,7 @@
 #include "common/OpContext.h"
 #include "common/PreparedGeometry.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "geos_c.h"
 #include "index/Index.h"
 #include "index/Meta.h"
@@ -528,7 +529,7 @@ PhyGISFunctionFilterExpr::EvalForIndexSegment() {
                 auto geometry_array =
                     static_cast<const milvus::proto::schema::GeometryArray*>(
                         &data_array->scalars().geometry_data());
-                const auto& valid_data = data_array->valid_data();
+                const auto& valid_data = GetFieldDataRowValidData(*data_array);
 
                 GEOSContextHandle_t local_ctx = GetThreadLocalGEOSContext();
                 for (size_t i = 0; i < hit_offsets.size(); ++i) {

internal/core/src/exec/expression/MatchExprTest.cpp

@@ -751,7 +751,7 @@ BuildNullableStructInsertData(const std::shared_ptr<Schema>& schema,
 
     const std::vector<bool> valid = {true, false, true, true, false};
     for (auto is_valid : valid) {
-        sub_field->add_valid_data(is_valid);
+        sub_field->mutable_scalars()->add_valid_data(is_valid);
     }
 
     auto append_row = [array_data](std::initializer_list<int32_t> values) {

internal/core/src/index/InvertedIndexArrayTest.cpp

@@ -426,7 +426,7 @@ TEST(ArrayInvertedIndexRegression,
         ASSERT_NE(array_field, nullptr);
         auto* array_data = array_field->mutable_scalars()->mutable_array_data();
         ASSERT_EQ(array_data->data_size(), row_count);
-        auto* valid_data = array_field->mutable_valid_data();
+        auto* valid_data = array_field->mutable_scalars()->mutable_valid_data();
         ASSERT_EQ(valid_data->size(), row_count);
 
         std::vector<boost::container::vector<int64_t>> index_arrays;

internal/core/src/index/TextMatchIndexTest.cpp

@@ -1030,8 +1030,10 @@ TEST(TextMatch, GrowingNaiveNullable) {
                        .mutable_scalars()
                        ->mutable_string_data()
                        ->mutable_data();
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     for (int64_t i = 0; i < N; i++) {
         str_col->at(i) = raw_str[i];
     }
@@ -1233,8 +1235,10 @@ TEST(TextMatch, SealedNaiveNullable) {
     for (int64_t i = 0; i < N; i++) {
         str_col->at(i) = raw_str[i];
     }
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     for (int64_t i = 0; i < N; i++) {
         str_col_valid->at(i) = raw_str_valid[i];
     }
@@ -1422,8 +1426,10 @@ TEST(TextMatch, GrowingJieBaNullable) {
     for (int64_t i = 0; i < N; i++) {
         str_col->at(i) = raw_str[i];
     }
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     for (int64_t i = 0; i < N; i++) {
         str_col_valid->at(i) = raw_str_valid[i];
     }
@@ -1604,8 +1610,10 @@ TEST(TextMatch, SealedJieBaNullable) {
     for (int64_t i = 0; i < N; i++) {
         str_col->at(i) = raw_str[i];
     }
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     for (int64_t i = 0; i < N; i++) {
         str_col_valid->at(i) = raw_str_valid[i];
     }
@@ -1708,8 +1716,10 @@ TEST(TextMatch, GrowingLoadData) {
     for (int64_t i = 0; i < N; i++) {
         str_col->at(i) = raw_str[i];
     }
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     for (int64_t i = 0; i < N; i++) {
         str_col_valid->at(i) = true;
     }
@@ -1746,8 +1756,10 @@ TEST(TextMatch, ConcurrentReadWriteWithNull) {
     int64_t N = 1000;
     uint64_t seed = 19190504;
     auto raw_data = DataGen(schema, N, seed);
-    auto str_col_valid =
-        raw_data.raw_->mutable_fields_data()->at(1).mutable_valid_data();
+    auto str_col_valid = raw_data.raw_->mutable_fields_data()
+                             ->at(1)
+                             .mutable_scalars()
+                             ->mutable_valid_data();
     auto str_col = raw_data.raw_->mutable_fields_data()
                        ->at(1)
                        .mutable_scalars()

internal/core/src/query/ExecPlanNodeVisitor.cpp

@@ -18,6 +18,7 @@
 #include <utility>
 
 #include "common/Tracer.h"
+#include "common/Utils.h"
 #include "common/protobuf_utils.h"
 #include "exec/Task.h"
 #include "fmt/core.h"
@@ -153,7 +154,7 @@ fillDataArrayFromColumnVector(const ColumnVectorPtr& column_vector,
     // Always copy validity data from ColumnVector
     // ColumnVector always tracks validity via valid_values_, so we should
     // always propagate it to ensure correctness for nullable fields
-    auto valid_data = data_array.mutable_valid_data();
+    auto valid_data = MutableFieldDataRowValidData(&data_array);
     const uint8_t* src_bitmap =
         static_cast<const uint8_t*>(column_vector->GetValidRawData());
     AssertInfo(src_bitmap,

internal/core/src/query/SearchOnSealedIndexBitsetLifetimeTest.cpp

@@ -33,6 +33,7 @@
 
 #include "common/BitsetView.h"
 #include "common/Chunk.h"
+#include "common/Utils.h"
 #include "common/IndexMeta.h"
 #include "common/QueryInfo.h"
 #include "common/QueryResult.h"
@@ -145,10 +146,11 @@ FindFieldData(const segcore::GeneratedData& dataset, FieldId field_id) {
 
 int64_t
 CountValidRows(const DataArray& data, int64_t total_count) {
-    if (data.valid_data_size() == 0) {
+    const auto& valid_data = GetFieldDataRowValidData(data);
+    if (valid_data.empty()) {
         return total_count;
     }
-    return std::count(data.valid_data().begin(), data.valid_data().end(), true);
+    return std::count(valid_data.begin(), valid_data.end(), true);
 }
 
 std::shared_ptr<ChunkedColumn>

internal/core/src/segcore/ChunkedSegmentSealedBinlogIndexTest.cpp

@@ -790,9 +790,10 @@ TEST(test_chunk_segment,
     auto null_result = sealed->bulk_subscript(
         nullptr, vec_field_id, null_offsets.data(), null_offsets.size());
 
-    ASSERT_EQ(null_result->valid_data_size(), null_offsets.size());
-    for (int i = 0; i < null_result->valid_data_size(); ++i) {
-        EXPECT_FALSE(null_result->valid_data(i));
+    const auto& null_valid_data = GetFieldDataRowValidData(*null_result);
+    ASSERT_EQ(null_valid_data.size(), null_offsets.size());
+    for (int i = 0; i < null_valid_data.size(); ++i) {
+        EXPECT_FALSE(null_valid_data[i]);
     }
     EXPECT_TRUE(null_result->vectors().float_vector().data().empty());
 
@@ -800,11 +801,12 @@ TEST(test_chunk_segment,
     auto result = sealed->bulk_subscript(
         nullptr, vec_field_id, offsets.data(), offsets.size());
 
-    ASSERT_EQ(result->valid_data_size(), offsets.size());
-    EXPECT_FALSE(result->valid_data(0));
-    EXPECT_TRUE(result->valid_data(1));
-    EXPECT_FALSE(result->valid_data(2));
-    EXPECT_TRUE(result->valid_data(3));
+    const auto& result_valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(result_valid_data.size(), offsets.size());
+    EXPECT_FALSE(result_valid_data[0]);
+    EXPECT_TRUE(result_valid_data[1]);
+    EXPECT_FALSE(result_valid_data[2]);
+    EXPECT_TRUE(result_valid_data[3]);
 
     const auto& returned = result->vectors().float_vector().data();
     ASSERT_EQ(returned.size(), 2 * dim);

internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp

@@ -5880,7 +5880,7 @@ ChunkedSegmentSealedImpl::get_raw_data(milvus::OpContext* op_ctx,
     }
 
     if (!field_meta.is_vector() && column->IsNullable()) {
-        auto dst = ret->mutable_valid_data()->mutable_data();
+        auto dst = MutableFieldDataRowValidData(ret.get())->mutable_data();
         column->BulkIsValid(
             op_ctx,
             [&](bool is_valid, size_t offset) { dst[offset] = is_valid; },
@@ -6277,7 +6277,7 @@ ChunkedSegmentSealedImpl::bulk_subscript(
                field_id.get());
     auto ret = fill_with_empty(field_id, count);
     if (column->IsNullable()) {
-        auto dst = ret->mutable_valid_data()->mutable_data();
+        auto dst = MutableFieldDataRowValidData(ret.get())->mutable_data();
         column->BulkIsValid(
             op_ctx,
             [&](bool is_valid, size_t offset) { dst[offset] = is_valid; },
@@ -9408,7 +9408,7 @@ ChunkedSegmentSealedImpl::ArrowToDataArray(
 
     // Populate valid_data for nullable fields so clients can identify nulls.
     if (field_meta.is_nullable()) {
-        auto* vd = data_array->mutable_valid_data();
+        auto* vd = MutableFieldDataRowValidData(data_array.get());
         vd->Reserve(size);
         for (int64_t i = 0; i < size; i++) {
             vd->Add(arr->IsValid(result_mapping[i]));

internal/core/src/segcore/ConcurrentVector.cpp

@@ -54,19 +54,20 @@ VectorBase::set_data_raw(ssize_t element_offset,
                 element_offset, VEC_FIELD_DATA(data, int8), element_count);
         } else if (field_meta.get_data_type() == DataType::VECTOR_ARRAY) {
             auto& vector_array = data->vectors().vector_array().data();
+            const auto& valid_data = GetFieldDataRowValidData(*data);
             std::vector<VectorArray> data_raw{};
             if (field_meta.is_nullable() &&
-                data->valid_data_size() == element_count) {
+                valid_data.size() == element_count) {
                 ssize_t valid_count = 0;
                 for (ssize_t i = 0; i < element_count; ++i) {
-                    if (data->valid_data(i)) {
+                    if (valid_data[i]) {
                         ++valid_count;
                     }
                 }
                 data_raw.reserve(valid_count);
                 if (vector_array.size() == element_count) {
                     for (ssize_t i = 0; i < element_count; ++i) {
-                        if (data->valid_data(i)) {
+                        if (valid_data[i]) {
                             data_raw.emplace_back(VectorArray(vector_array[i]));
                         }
                     }

internal/core/src/segcore/ConcurrentVector.h

@@ -81,7 +81,8 @@ class ThreadSafeValidData {
         std::unique_lock<std::shared_mutex> lck(mutex_);
         if (field_meta.is_nullable()) {
             reserve_to(length_ + num_rows);
-            write_from(length_, num_rows, data->valid_data().data());
+            write_from(
+                length_, num_rows, GetFieldDataRowValidData(*data).data());
             length_ += num_rows;
         }
     }
@@ -110,7 +111,9 @@ class ThreadSafeValidData {
                        element_offset,
                        length_);
             reserve_to(end);
-            write_from(element_offset, num_rows, data->valid_data().data());
+            write_from(element_offset,
+                       num_rows,
+                       GetFieldDataRowValidData(*data).data());
             length_ = std::max(length_, end);
         }
     }

internal/core/src/segcore/FieldIndexing.cpp

@@ -685,7 +685,7 @@ ScalarFieldIndexing<T>::AppendSegmentIndex(int64_t reserved_offset,
                 stream_data->scalars().has_geometry_data()) {
                 const auto& geometry_array =
                     stream_data->scalars().geometry_data();
-                const auto& valid_data = stream_data->valid_data();
+                const auto& valid_data = GetFieldDataRowValidData(*stream_data);
 
                 // Create accessor for DataArray
                 auto accessor = [&geometry_array, &valid_data](

internal/core/src/segcore/SegmentGrowingImpl.cpp

@@ -51,6 +51,7 @@
 #include "common/Schema.h"
 #include "common/Span.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "common/VectorArray.h"
 #include "glog/logging.h"
 #include "index/Index.h"
@@ -232,6 +233,7 @@ ExtractArrayLengths(const proto::schema::FieldData& field_data,
                     int64_t num_rows,
                     int32_t* array_lengths) {
     auto data_type = field_meta.get_data_type();
+    const auto& valid_data = GetFieldDataRowValidData(field_data);
     if (data_type == DataType::VECTOR_ARRAY) {
         const auto& vector_array = field_data.vectors().vector_array();
         int64_t dim = field_meta.get_dim();
@@ -239,15 +241,15 @@ ExtractArrayLengths(const proto::schema::FieldData& field_data,
         bool compact_nullable = false;
         if (field_meta.is_nullable()) {
             int64_t valid_count = 0;
-            if (field_data.valid_data_size() == num_rows) {
+            if (valid_data.size() == num_rows) {
                 for (int64_t i = 0; i < num_rows; ++i) {
-                    if (field_data.valid_data(i)) {
+                    if (valid_data[i]) {
                         ++valid_count;
                     }
                 }
             }
             auto dense_aligned = vector_array.data_size() == num_rows;
-            compact_nullable = field_data.valid_data_size() == num_rows &&
+            compact_nullable = valid_data.size() == num_rows &&
                                vector_array.data_size() == valid_count;
             AssertInfo(
                 dense_aligned || compact_nullable,
@@ -257,7 +259,7 @@ ExtractArrayLengths(const proto::schema::FieldData& field_data,
                 "got data_size {}, valid_data_size {}, num_rows {}, "
                 "valid_count {}",
                 vector_array.data_size(),
-                field_data.valid_data_size(),
+                valid_data.size(),
                 num_rows,
                 valid_count);
             compact_nullable = compact_nullable && !dense_aligned;
@@ -269,11 +271,10 @@ ExtractArrayLengths(const proto::schema::FieldData& field_data,
                        num_rows);
         }
         int64_t physical_row = 0;
-        auto has_valid_data = field_data.valid_data_size() == num_rows;
+        auto has_valid_data = valid_data.size() == num_rows;
 
         for (int i = 0; i < num_rows; ++i) {
-            if (field_meta.is_nullable() && has_valid_data &&
-                !field_data.valid_data(i)) {
+            if (field_meta.is_nullable() && has_valid_data && !valid_data[i]) {
                 array_lengths[i] = 0;
                 continue;
             }
@@ -326,8 +327,7 @@ ExtractArrayLengths(const proto::schema::FieldData& field_data,
     }
 
     // Handle nullable fields
-    if (field_meta.is_nullable() && field_data.valid_data_size() > 0) {
-        const auto& valid_data = field_data.valid_data();
+    if (field_meta.is_nullable() && !valid_data.empty()) {
         for (int i = 0; i < num_rows; ++i) {
             if (!valid_data[i]) {
                 array_lengths[i] = 0;  // null → empty array
@@ -840,13 +840,10 @@ SegmentGrowingImpl::Insert(int64_t reserved_offset,
                     .string_data()
                     .data()
                     .end());
-            FixedVector<bool> texts_valid_data(
-                insert_record_proto->fields_data(data_offset)
-                    .valid_data()
-                    .begin(),
-                insert_record_proto->fields_data(data_offset)
-                    .valid_data()
-                    .end());
+            const auto& row_valid_data = GetFieldDataRowValidData(
+                insert_record_proto->fields_data(data_offset));
+            FixedVector<bool> texts_valid_data(row_valid_data.begin(),
+                                               row_valid_data.end());
             AddTexts(field_id,
                      texts.data(),
                      texts_valid_data.data(),
@@ -1705,7 +1702,7 @@ SegmentGrowingImpl::bulk_subscript(
     auto result = CreateEmptyScalarDataArray(count, field_meta);
     if (field_meta.is_nullable()) {
         auto valid_data_ptr = insert_record_.get_valid_data(field_id);
-        auto res = result->mutable_valid_data()->mutable_data();
+        auto res = MutableFieldDataRowValidData(result.get())->mutable_data();
         for (int64_t i = 0; i < count; ++i) {
             auto offset = seg_offsets[i];
             res[i] = valid_data_ptr->is_valid(offset);
@@ -1828,7 +1825,7 @@ SegmentGrowingImpl::bulk_subscript(milvus::OpContext* op_ctx,
     auto result = CreateEmptyScalarDataArray(count, field_meta);
     if (field_meta.is_nullable()) {
         auto valid_data_ptr = insert_record_.get_valid_data(field_id);
-        auto res = result->mutable_valid_data()->mutable_data();
+        auto res = MutableFieldDataRowValidData(result.get())->mutable_data();
         for (int64_t i = 0; i < count; ++i) {
             auto offset = seg_offsets[i];
             res[i] = valid_data_ptr->is_valid(offset);
@@ -3240,7 +3237,7 @@ SegmentGrowingImpl::BuildGeometryCacheForInsert(FieldId field_id,
 
         // Process geometry data from DataArray
         const auto& geometry_data = data_array->scalars().geometry_data();
-        const auto& valid_data = data_array->valid_data();
+        const auto& valid_data = GetFieldDataRowValidData(*data_array);
 
         for (int64_t i = 0; i < num_rows; ++i) {
             if (valid_data.empty() ||

internal/core/src/segcore/SegmentGrowingIndexTest.cpp

@@ -426,8 +426,9 @@ class GrowingIndexRawOwnershipTest : public ::testing::Test {
     FieldDataPtr
     CreateNullableFloatFieldData(const DataArray& data) const {
         std::vector<uint8_t> valid_bitmap((row_count + 7) / 8, 0);
+        const auto& valid_data = GetFieldDataRowValidData(data);
         for (int64_t i = 0; i < row_count; ++i) {
-            if (data.valid_data(i)) {
+            if (valid_data[i]) {
                 valid_bitmap[i / 8] |= uint8_t{1} << (i % 8);
             }
         }
@@ -444,9 +445,11 @@ class GrowingIndexRawOwnershipTest : public ::testing::Test {
     void
     AssertNullableFloatDataEqual(const DataArray& actual,
                                  const DataArray& expected) const {
-        ASSERT_EQ(actual.valid_data_size(), expected.valid_data_size());
-        for (int i = 0; i < actual.valid_data_size(); ++i) {
-            EXPECT_EQ(actual.valid_data(i), expected.valid_data(i));
+        const auto& actual_valid_data = GetFieldDataRowValidData(actual);
+        const auto& expected_valid_data = GetFieldDataRowValidData(expected);
+        ASSERT_EQ(actual_valid_data.size(), expected_valid_data.size());
+        for (int i = 0; i < actual_valid_data.size(); ++i) {
+            EXPECT_EQ(actual_valid_data[i], expected_valid_data[i]);
         }
 
         const auto& actual_values = actual.vectors().float_vector().data();

internal/core/src/segcore/SegmentGrowingTest.cpp

@@ -568,22 +568,22 @@ TEST_P(GrowingTest, FillData) {
         EXPECT_EQ(float_array_result->scalars().array_data().data_size(),
                   num_inserted);
 
-        EXPECT_EQ(bool_result->valid_data_size(), 0);
-        EXPECT_EQ(int8_result->valid_data_size(), 0);
-        EXPECT_EQ(int16_result->valid_data_size(), 0);
-        EXPECT_EQ(int32_result->valid_data_size(), 0);
-        EXPECT_EQ(int64_result->valid_data_size(), 0);
-        EXPECT_EQ(float_result->valid_data_size(), 0);
-        EXPECT_EQ(double_result->valid_data_size(), 0);
-        EXPECT_EQ(timestamptz_result->valid_data_size(), 0);
-        EXPECT_EQ(varchar_result->valid_data_size(), 0);
-        EXPECT_EQ(json_result->valid_data_size(), 0);
-        EXPECT_EQ(int_array_result->valid_data_size(), 0);
-        EXPECT_EQ(long_array_result->valid_data_size(), 0);
-        EXPECT_EQ(bool_array_result->valid_data_size(), 0);
-        EXPECT_EQ(string_array_result->valid_data_size(), 0);
-        EXPECT_EQ(double_array_result->valid_data_size(), 0);
-        EXPECT_EQ(float_array_result->valid_data_size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*bool_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int8_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int16_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int32_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int64_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*float_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*double_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*timestamptz_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*varchar_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*json_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int_array_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*long_array_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*bool_array_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*string_array_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*double_array_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*float_array_result).size(), 0);
     }
 }
 
@@ -712,21 +712,30 @@ TEST(Growing, FillNullableData) {
                   num_inserted);
         EXPECT_EQ(float_array_result->scalars().array_data().data_size(),
                   num_inserted);
-        EXPECT_EQ(bool_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(int8_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(int16_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(int32_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(float_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(double_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(timestamptz_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(varchar_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(json_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(int_array_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(long_array_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(bool_array_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(string_array_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(double_array_result->valid_data_size(), num_inserted);
-        EXPECT_EQ(float_array_result->valid_data_size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*bool_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*int8_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*int16_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*int32_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*float_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*double_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*timestamptz_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*varchar_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*json_result).size(), num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*int_array_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*long_array_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*bool_array_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*string_array_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*double_array_result).size(),
+                  num_inserted);
+        EXPECT_EQ(GetFieldDataRowValidData(*float_array_result).size(),
+                  num_inserted);
     }
 }
 
@@ -1127,8 +1136,9 @@ TEST_P(GrowingTest, FillVectorArrayData) {
             }
         }
 
-        EXPECT_EQ(int64_result->valid_data_size(), 0);
-        EXPECT_EQ(array_float_vector_result->valid_data_size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*int64_result).size(), 0);
+        EXPECT_EQ(GetFieldDataRowValidData(*array_float_vector_result).size(),
+                  0);
     }
 }
 
@@ -1185,9 +1195,9 @@ TEST(GrowingTest, QueryNullableVectorArrayUsesPhysicalOffsets) {
     auto array_data = insert_record_proto->add_fields_data();
     array_data->set_field_id(array_float_vector.get());
     array_data->set_type(proto::schema::DataType::ArrayOfVector);
-    array_data->add_valid_data(false);
-    array_data->add_valid_data(true);
     array_data->mutable_vectors()->set_dim(4);
+    array_data->mutable_vectors()->add_valid_data(false);
+    array_data->mutable_vectors()->add_valid_data(true);
     auto vector_array = array_data->mutable_vectors()->mutable_vector_array();
     vector_array->set_dim(4);
     vector_array->set_element_type(proto::schema::DataType::FloatVector);
@@ -1213,9 +1223,10 @@ TEST(GrowingTest, QueryNullableVectorArrayUsesPhysicalOffsets) {
             nullptr, array_float_vector, offsets.data(), offsets.size()));
     ASSERT_NE(result, nullptr);
 
-    ASSERT_EQ(result->valid_data_size(), 2);
-    EXPECT_FALSE(result->valid_data(0));
-    EXPECT_TRUE(result->valid_data(1));
+    const auto& result_valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(result_valid_data.size(), 2);
+    EXPECT_FALSE(result_valid_data[0]);
+    EXPECT_TRUE(result_valid_data[1]);
     ASSERT_EQ(result->vectors().vector_array().data_size(), 2);
     EXPECT_TRUE(
         result->vectors().vector_array().data(0).float_vector().data().empty());
@@ -1256,9 +1267,9 @@ TEST(GrowingTest, QueryNullableVectorArrayStoresRowDenseInputByValidData) {
     auto array_data = insert_record_proto->add_fields_data();
     array_data->set_field_id(array_float_vector.get());
     array_data->set_type(proto::schema::DataType::ArrayOfVector);
-    array_data->add_valid_data(false);
-    array_data->add_valid_data(true);
     array_data->mutable_vectors()->set_dim(4);
+    array_data->mutable_vectors()->add_valid_data(false);
+    array_data->mutable_vectors()->add_valid_data(true);
     auto vector_array = array_data->mutable_vectors()->mutable_vector_array();
     vector_array->set_dim(4);
     vector_array->set_element_type(proto::schema::DataType::FloatVector);
@@ -1287,9 +1298,10 @@ TEST(GrowingTest, QueryNullableVectorArrayStoresRowDenseInputByValidData) {
             nullptr, array_float_vector, offsets.data(), offsets.size()));
     ASSERT_NE(result, nullptr);
 
-    ASSERT_EQ(result->valid_data_size(), 2);
-    EXPECT_FALSE(result->valid_data(0));
-    EXPECT_TRUE(result->valid_data(1));
+    const auto& result_valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(result_valid_data.size(), 2);
+    EXPECT_FALSE(result_valid_data[0]);
+    EXPECT_TRUE(result_valid_data[1]);
     ASSERT_EQ(result->vectors().vector_array().data_size(), 2);
     EXPECT_TRUE(
         result->vectors().vector_array().data(0).float_vector().data().empty());
@@ -1624,12 +1636,12 @@ TEST(Growing, TestMaskWithNullableTTLField) {
         // Add valid_data for nullable field
         // Note: valid_data[i] = false means null, valid_data[i] = true means non-null
         for (size_t i = 0; i < valid_data.size(); ++i) {
-            field_data->add_valid_data(valid_data[i]);
+            scalars->add_valid_data(valid_data[i]);
         }
         // Verify valid_data was added correctly
-        ASSERT_EQ(field_data->valid_data_size(), test_data_count)
+        ASSERT_EQ(scalars->valid_data_size(), test_data_count)
             << "valid_data size mismatch: expected " << test_data_count
-            << ", got " << field_data->valid_data_size();
+            << ", got " << scalars->valid_data_size();
     }
 
     // Insert data

internal/core/src/segcore/SegmentInterface.cpp

@@ -838,7 +838,7 @@ SegmentInternalInterface::bulk_subscript_not_exist_field(
         auto create_count = IsVectorArrayDataType(data_type) ? count : 0;
         auto result = CreateEmptyVectorDataArray(create_count, field_meta);
 
-        auto valid_data = result->mutable_valid_data();
+        auto valid_data = MutableFieldDataRowValidData(result.get());
         for (int64_t i = 0; i < count; ++i) {
             valid_data->Add(false);
         }
@@ -848,7 +848,8 @@ SegmentInternalInterface::bulk_subscript_not_exist_field(
     auto result = CreateEmptyScalarDataArray(count, field_meta);
     if (field_meta.default_value().has_value()) {
         if (field_meta.is_nullable()) {
-            auto res = result->mutable_valid_data()->mutable_data();
+            auto res =
+                MutableFieldDataRowValidData(result.get())->mutable_data();
             for (int64_t i = 0; i < count; ++i) {
                 res[i] = true;
             }

internal/core/src/segcore/Utils.cpp

@@ -373,13 +373,12 @@ CreateEmptyScalarDataArray(int64_t count, const FieldMeta& field_meta) {
     data_array->set_type(static_cast<milvus::proto::schema::DataType>(
         field_meta.get_data_type()));
 
-    if (field_meta.is_nullable()) {
-        data_array->mutable_valid_data()->Resize(count, false);
-    }
-
     auto scalar_array = data_array->mutable_scalars();
     SetUpScalarFieldData(
         scalar_array, data_type, field_meta.get_element_type(), count);
+    if (field_meta.is_nullable()) {
+        scalar_array->mutable_valid_data()->Resize(count, false);
+    }
     return data_array;
 }
 
@@ -391,11 +390,11 @@ CreateScalarDataArray(DataArray& data_array,
                       bool nullable) {
     data_array.set_type(
         static_cast<milvus::proto::schema::DataType>(data_type));
-    if (nullable) {
-        data_array.mutable_valid_data()->Resize(count, false);
-    }
     auto scalar_array = data_array.mutable_scalars();
     SetUpScalarFieldData(scalar_array, data_type, element_type, count);
+    if (nullable) {
+        scalar_array->mutable_valid_data()->Resize(count, false);
+    }
 }
 
 void
@@ -570,7 +569,7 @@ CreateEmptyVectorDataArray(int64_t count,
                              : count;
     auto data_array = CreateEmptyVectorDataArray(data_count, field_meta);
     if (field_meta.is_nullable() && valid_data != nullptr) {
-        auto obj = data_array->mutable_valid_data();
+        auto obj = MutableFieldDataRowValidData(data_array.get());
         auto valid_data_bool = reinterpret_cast<const bool*>(valid_data);
         obj->Add(valid_data_bool, valid_data_bool + count);
     }
@@ -587,18 +586,17 @@ CreateScalarDataArrayFrom(const void* data_raw,
     data_array->set_field_id(field_meta.get_id().get());
     data_array->set_type(static_cast<milvus::proto::schema::DataType>(
         field_meta.get_data_type()));
+    auto scalar_array = data_array->mutable_scalars();
     if (field_meta.is_nullable() && valid_data != nullptr) {
         auto valid_data_ = reinterpret_cast<const bool*>(valid_data);
-        auto obj = data_array->mutable_valid_data();
+        auto obj = scalar_array->mutable_valid_data();
         obj->Add(valid_data_, valid_data_ + count);
     } else {
         FixedVector<bool> always_valid(count, true);
-        auto obj = data_array->mutable_valid_data();
+        auto obj = scalar_array->mutable_valid_data();
         obj->Add(reinterpret_cast<const bool*>(always_valid.data()),
                  reinterpret_cast<const bool*>(always_valid.data()) + count);
     }
-
-    auto scalar_array = data_array->mutable_scalars();
     switch (data_type) {
         case DataType::BOOL: {
             auto data = reinterpret_cast<const bool*>(data_raw);
@@ -835,7 +833,7 @@ CreateVectorDataArrayFrom(const void* data_raw,
     auto data_array =
         CreateVectorDataArrayFrom(data_raw, valid_count, field_meta);
     if (field_meta.is_nullable() && valid_data != nullptr) {
-        auto obj = data_array->mutable_valid_data();
+        auto obj = MutableFieldDataRowValidData(data_array.get());
         auto valid_data_bool = reinterpret_cast<const bool*>(valid_data);
         obj->Add(valid_data_bool, valid_data_bool + count);
     }
@@ -857,7 +855,7 @@ CreateDataArrayFrom(const void* data_raw,
     auto data_array = CreateVectorDataArrayFrom(data_raw, count, field_meta);
     if (field_meta.get_data_type() == DataType::VECTOR_ARRAY &&
         field_meta.is_nullable() && valid_data != nullptr) {
-        auto obj = data_array->mutable_valid_data();
+        auto obj = MutableFieldDataRowValidData(data_array.get());
         auto valid_data_bool = reinterpret_cast<const bool*>(valid_data);
         obj->Add(valid_data_bool, valid_data_bool + count);
     }
@@ -882,6 +880,13 @@ MergeDataArray(std::vector<MergeBase>& merge_bases,
     auto nullable = field_meta.is_nullable();
     data_array->set_type(static_cast<milvus::proto::schema::DataType>(
         field_meta.get_data_type()));
+    if (field_meta.is_vector()) {
+        data_array->mutable_vectors();
+    } else {
+        data_array->mutable_scalars();
+    }
+    auto* dst_valid_data =
+        nullable ? MutableFieldDataRowValidData(data_array.get()) : nullptr;
 
     for (auto& merge_base : merge_bases) {
         auto src_field_data = merge_base.get_field_data(field_meta.get_id());
@@ -891,10 +896,9 @@ MergeDataArray(std::vector<MergeBase>& merge_bases,
         if (field_meta.is_vector()) {
             bool is_valid = true;
             if (nullable) {
-                auto data = src_field_data->valid_data().data();
-                auto obj = data_array->mutable_valid_data();
+                const auto& data = GetFieldDataRowValidData(*src_field_data);
                 is_valid = data[src_offset];
-                *(obj->Add()) = is_valid;
+                *(dst_valid_data->Add()) = is_valid;
             }
 
             if (!is_valid) {
@@ -980,9 +984,8 @@ MergeDataArray(std::vector<MergeBase>& merge_bases,
         }
 
         if (nullable) {
-            auto data = src_field_data->valid_data().data();
-            auto obj = data_array->mutable_valid_data();
-            *(obj->Add()) = data[src_offset];
+            const auto& data = GetFieldDataRowValidData(*src_field_data);
+            *(dst_valid_data->Add()) = data[src_offset];
         }
 
         auto scalar_array = data_array->mutable_scalars();
@@ -1301,8 +1304,8 @@ ReverseDataFromIndex(const index::IndexBase* index,
     }
 
     if (nullable) {
-        *(data_array->mutable_valid_data()) = {valid_data.begin(),
-                                               valid_data.end()};
+        *(MutableFieldDataRowValidData(data_array.get())) = {valid_data.begin(),
+                                                             valid_data.end()};
     }
 
     return data_array;

internal/core/src/segcore/UtilsTest.cpp

@@ -29,6 +29,7 @@
 #include "common/SealedOffsetMapping.h"
 #include "common/Schema.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "common/protobuf_utils.h"
 #include "gtest/gtest.h"
 #include "knowhere/comp/index_param.h"
@@ -127,18 +128,20 @@ TEST(Util_Segcore, CreateEmptyVectorDataArrayForNullableVectors) {
 
     auto dense_result =
         CreateEmptyVectorDataArray(3, 0, valid_data.data(), (*schema)[vec]);
-    ASSERT_EQ(dense_result->valid_data().size(), 3);
-    ASSERT_FALSE(dense_result->valid_data(0));
-    ASSERT_FALSE(dense_result->valid_data(1));
-    ASSERT_FALSE(dense_result->valid_data(2));
+    const auto& dense_valid_data = GetFieldDataRowValidData(*dense_result);
+    ASSERT_EQ(dense_valid_data.size(), 3);
+    ASSERT_FALSE(dense_valid_data[0]);
+    ASSERT_FALSE(dense_valid_data[1]);
+    ASSERT_FALSE(dense_valid_data[2]);
     ASSERT_EQ(dense_result->vectors().float_vector().data_size(), 0);
 
     auto sparse_result = CreateEmptyVectorDataArray(
         3, 0, valid_data.data(), (*schema)[sparse_vec]);
-    ASSERT_EQ(sparse_result->valid_data().size(), 3);
-    ASSERT_FALSE(sparse_result->valid_data(0));
-    ASSERT_FALSE(sparse_result->valid_data(1));
-    ASSERT_FALSE(sparse_result->valid_data(2));
+    const auto& sparse_valid_data = GetFieldDataRowValidData(*sparse_result);
+    ASSERT_EQ(sparse_valid_data.size(), 3);
+    ASSERT_FALSE(sparse_valid_data[0]);
+    ASSERT_FALSE(sparse_valid_data[1]);
+    ASSERT_FALSE(sparse_valid_data[2]);
     ASSERT_EQ(sparse_result->vectors().data_case(),
               proto::schema::VectorField::kSparseFloatVector);
     ASSERT_EQ(sparse_result->vectors().sparse_float_vector().contents_size(),
@@ -175,8 +178,9 @@ TEST(Util_Segcore, CreateVectorDataArrayFromNullableVectors) {
     auto result = CreateVectorDataArrayFrom(
         data.data(), valid_flags.get(), total_count, valid_count, field_meta);
 
-    ASSERT_TRUE(result->valid_data().size() > 0);
-    ASSERT_EQ(result->valid_data().size(), total_count);
+    const auto& result_valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_FALSE(result_valid_data.empty());
+    ASSERT_EQ(result_valid_data.size(), total_count);
     ASSERT_EQ(result->vectors().float_vector().data_size(), valid_count * dim);
 }
 
@@ -227,15 +231,16 @@ TEST(Util_Segcore, MergeDataArrayWithNullableVectors) {
 
     auto merged_result = MergeDataArray(merge_bases, field_meta);
 
-    ASSERT_TRUE(merged_result->valid_data().size() > 0);
-    ASSERT_EQ(merged_result->valid_data().size(), 5);
+    const auto& merged_valid_data = GetFieldDataRowValidData(*merged_result);
+    ASSERT_FALSE(merged_valid_data.empty());
+    ASSERT_EQ(merged_valid_data.size(), 5);
     ASSERT_EQ(merged_result->vectors().float_vector().data_size(), 5 * dim);
 
-    ASSERT_TRUE(merged_result->valid_data(0));
-    ASSERT_TRUE(merged_result->valid_data(1));
-    ASSERT_TRUE(merged_result->valid_data(2));
-    ASSERT_TRUE(merged_result->valid_data(3));
-    ASSERT_TRUE(merged_result->valid_data(4));
+    ASSERT_TRUE(merged_valid_data[0]);
+    ASSERT_TRUE(merged_valid_data[1]);
+    ASSERT_TRUE(merged_valid_data[2]);
+    ASSERT_TRUE(merged_valid_data[3]);
+    ASSERT_TRUE(merged_valid_data[4]);
 }
 
 TEST(Util_Segcore, MergeDataArrayWithNullableByteVectorsAppendsRows) {
@@ -297,11 +302,13 @@ TEST(Util_Segcore, MergeDataArrayWithNullableByteVectorsAppendsRows) {
 
         auto merged_result = MergeDataArray(merge_bases, field_meta);
 
-        ASSERT_EQ(merged_result->valid_data().size(), total_count);
-        EXPECT_TRUE(merged_result->valid_data(0));
-        EXPECT_FALSE(merged_result->valid_data(1));
-        EXPECT_TRUE(merged_result->valid_data(2));
-        EXPECT_TRUE(merged_result->valid_data(3));
+        const auto& merged_valid_data =
+            GetFieldDataRowValidData(*merged_result);
+        ASSERT_EQ(merged_valid_data.size(), total_count);
+        EXPECT_TRUE(merged_valid_data[0]);
+        EXPECT_FALSE(merged_valid_data[1]);
+        EXPECT_TRUE(merged_valid_data[2]);
+        EXPECT_TRUE(merged_valid_data[3]);
 
         std::string actual;
         switch (test_case.data_type) {
@@ -515,10 +522,11 @@ TEST(Util_Segcore, MergeDataArrayWithNullableVectorArrayUsesLogicalOffsets) {
 
     auto merged_result = MergeDataArray(merge_bases, field_meta);
 
-    ASSERT_EQ(merged_result->valid_data_size(), 3);
-    EXPECT_FALSE(merged_result->valid_data(0));
-    EXPECT_TRUE(merged_result->valid_data(1));
-    EXPECT_TRUE(merged_result->valid_data(2));
+    const auto& merged_valid_data = GetFieldDataRowValidData(*merged_result);
+    ASSERT_EQ(merged_valid_data.size(), 3);
+    EXPECT_FALSE(merged_valid_data[0]);
+    EXPECT_TRUE(merged_valid_data[1]);
+    EXPECT_TRUE(merged_valid_data[2]);
 
     const auto& result_rows = merged_result->vectors().vector_array().data();
     ASSERT_EQ(result_rows.size(), 3);

internal/core/src/segcore/flush_growing_segment_test.cpp

@@ -2974,10 +2974,11 @@ TEST_F(FlushGrowingSegmentTest, FlushNullableEmbListMixedRows) {
         *rows->Add() = expected_empty;
         *rows->Add() = expected_empty;
         *rows->Add() = expected_target;
-        field_data->mutable_valid_data()->Clear();
-        field_data->add_valid_data(false);
-        field_data->add_valid_data(true);
-        field_data->add_valid_data(true);
+        auto* valid_data = field_data->mutable_vectors()->mutable_valid_data();
+        valid_data->Clear();
+        valid_data->Add(false);
+        valid_data->Add(true);
+        valid_data->Add(true);
         break;
     }
 

internal/core/src/segcore/search_result_export_c.cpp

@@ -30,6 +30,7 @@
 
 #include "common/EasyAssert.h"
 #include "common/FieldMeta.h"
+#include "common/Utils.h"
 #include "log/Log.h"
 #include "common/QueryResult.h"
 #include "common/Types.h"
@@ -333,19 +334,19 @@ BuildFixedWidthArray(const DataContainer& data,
                "field data length {} is smaller than expected row count {}",
                data.size(),
                total_valid);
-    const bool has_valid_data = field_data.valid_data_size() > 0;
+    const auto& valid_data = milvus::GetFieldDataRowValidData(field_data);
+    const bool has_valid_data = !valid_data.empty();
     if (has_valid_data) {
-        AssertInfo(
-            static_cast<size_t>(field_data.valid_data_size()) == total_valid,
-            "valid_data length {} does not match expected row count {}",
-            field_data.valid_data_size(),
-            total_valid);
+        AssertInfo(static_cast<size_t>(valid_data.size()) == total_valid,
+                   "valid_data length {} does not match expected row count {}",
+                   valid_data.size(),
+                   total_valid);
     }
 
     BuilderType builder;
     ARROW_RETURN_NOT_OK(builder.Reserve(total_valid));
     for (size_t i = 0; i < total_valid; ++i) {
-        if (has_valid_data && !field_data.valid_data(i)) {
+        if (has_valid_data && !valid_data[i]) {
             ARROW_RETURN_NOT_OK(builder.AppendNull());
             continue;
         }
@@ -366,19 +367,19 @@ BuildVarLenArray(const DataContainer& data,
                "field data length {} is smaller than expected row count {}",
                data.size(),
                total_valid);
-    const bool has_valid_data = field_data.valid_data_size() > 0;
+    const auto& valid_data = milvus::GetFieldDataRowValidData(field_data);
+    const bool has_valid_data = !valid_data.empty();
     if (has_valid_data) {
-        AssertInfo(
-            static_cast<size_t>(field_data.valid_data_size()) == total_valid,
-            "valid_data length {} does not match expected row count {}",
-            field_data.valid_data_size(),
-            total_valid);
+        AssertInfo(static_cast<size_t>(valid_data.size()) == total_valid,
+                   "valid_data length {} does not match expected row count {}",
+                   valid_data.size(),
+                   total_valid);
     }
 
     BuilderType builder;
     ARROW_RETURN_NOT_OK(builder.Reserve(total_valid));
     for (size_t i = 0; i < total_valid; ++i) {
-        if (has_valid_data && !field_data.valid_data(i)) {
+        if (has_valid_data && !valid_data[i]) {
             ARROW_RETURN_NOT_OK(builder.AppendNull());
             continue;
         }
@@ -1044,14 +1045,16 @@ FillOutputFieldsOrderedImpl(CSearchResult* search_results,
                     continue;
                 }
                 auto& field_data = it->second;
-                if (field_data->valid_data_size() == 0) {
+                const auto& valid_data =
+                    milvus::GetFieldDataRowValidData(*field_data);
+                if (valid_data.empty()) {
                     continue;
                 }
                 int64_t valid_idx = 0;
                 for (size_t i = 0; i < seg_res.result_positions.size(); i++) {
                     auto pos = seg_res.result_positions[i];
                     result_pairs[pos].setValidDataOffset(field_id, valid_idx);
-                    if (field_data->valid_data(i)) {
+                    if (valid_data[i]) {
                         valid_idx++;
                     }
                 }

internal/core/src/segcore/segment_c.cpp

@@ -1181,12 +1181,11 @@ BuildFixedWidthVectorArrayFromBulkSubscript(
             return arrow::Status::Invalid("not a fixed-width vector field");
     }
 
-    bool has_valid_data = data_array.valid_data_size() > 0;
-    int64_t valid_count = has_valid_data
-                              ? std::count(data_array.valid_data().begin(),
-                                           data_array.valid_data().end(),
-                                           true)
-                              : num_rows;
+    const auto& valid_data = milvus::GetFieldDataRowValidData(data_array);
+    bool has_valid_data = !valid_data.empty();
+    int64_t valid_count =
+        has_valid_data ? std::count(valid_data.begin(), valid_data.end(), true)
+                       : num_rows;
     if (data_size != valid_count * byte_width) {
         return arrow::Status::Invalid(fmt::format(
             "bulk_subscript vector payload size mismatch, field={}, bytes={}, "
@@ -1203,7 +1202,7 @@ BuildFixedWidthVectorArrayFromBulkSubscript(
         auto bytes = static_cast<const uint8_t*>(data);
         int64_t physical = 0;
         for (int64_t i = 0; i < num_rows; i++) {
-            bool is_valid = !has_valid_data || data_array.valid_data(i);
+            bool is_valid = !has_valid_data || valid_data[i];
             if (!is_valid) {
                 ARROW_RETURN_NOT_OK(builder.AppendNull());
                 continue;
@@ -1230,12 +1229,11 @@ arrow::Result<std::shared_ptr<arrow::Array>>
 BuildSparseVectorArrayFromBulkSubscript(
     const milvus::proto::schema::FieldData& data_array, int64_t num_rows) {
     const auto& sparse = data_array.vectors().sparse_float_vector();
-    bool has_valid_data = data_array.valid_data_size() > 0;
-    int64_t valid_count = has_valid_data
-                              ? std::count(data_array.valid_data().begin(),
-                                           data_array.valid_data().end(),
-                                           true)
-                              : num_rows;
+    const auto& valid_data = milvus::GetFieldDataRowValidData(data_array);
+    bool has_valid_data = !valid_data.empty();
+    int64_t valid_count =
+        has_valid_data ? std::count(valid_data.begin(), valid_data.end(), true)
+                       : num_rows;
     if (sparse.contents_size() != valid_count) {
         return arrow::Status::Invalid(
             fmt::format("bulk_subscript sparse payload size mismatch, rows={}, "
@@ -1249,7 +1247,7 @@ BuildSparseVectorArrayFromBulkSubscript(
     ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
     int64_t physical = 0;
     for (int64_t i = 0; i < num_rows; i++) {
-        bool is_valid = !has_valid_data || data_array.valid_data(i);
+        bool is_valid = !has_valid_data || valid_data[i];
         if (!is_valid) {
             ARROW_RETURN_NOT_OK(builder.AppendNull());
             continue;

internal/core/unittest/test_bloom_filter_expr.cpp

@@ -684,9 +684,10 @@ class BloomFilterExprEvalTest : public ::testing::Test {
             for (const auto& s : rows) {
                 jd->Add(std::string(s));
             }
-            fd.mutable_valid_data()->Clear();
+            auto* valid_data = fd.mutable_scalars()->mutable_valid_data();
+            valid_data->Clear();
             for (bool v : valid) {
-                fd.mutable_valid_data()->Add(v);
+                valid_data->Add(v);
             }
         }
         ASSERT_TRUE(found) << "json field not present in generated dataset";

internal/core/unittest/test_element_filter.cpp

@@ -29,6 +29,7 @@
 #include "common/IndexMeta.h"
 #include "common/QueryResult.h"
 #include "common/Schema.h"
+#include "common/Utils.h"
 #include "common/TracerBase.h"
 #include "common/Types.h"
 #include "common/VectorTrait.h"
@@ -2470,10 +2471,11 @@ TEST(ElementFilter, GrowingNullableArrayTailChunkUsesActiveRows) {
     auto array_data = insert_record_proto->add_fields_data();
     array_data->set_field_id(int_array_fid.get());
     array_data->set_type(proto::schema::DataType::Array);
-    array_data->add_valid_data(true);
-    array_data->add_valid_data(false);
-    array_data->add_valid_data(true);
-    auto arrays = array_data->mutable_scalars()->mutable_array_data();
+    auto* array_scalars = array_data->mutable_scalars();
+    array_scalars->add_valid_data(true);
+    array_scalars->add_valid_data(false);
+    array_scalars->add_valid_data(true);
+    auto arrays = array_scalars->mutable_array_data();
     arrays->set_element_type(proto::schema::DataType::Int32);
     auto row0 = arrays->mutable_data()->Add();
     row0->mutable_int_data()->mutable_data()->Add(10);
@@ -3773,7 +3775,7 @@ MakeNullableElementSearchFixture() {
             }
         }
 
-        auto* valid_data = fd->mutable_valid_data();
+        auto* valid_data = fd->mutable_vectors()->mutable_valid_data();
         valid_data->Clear();
         for (int row = 0; row < kNullableElemN; ++row) {
             valid_data->Add(true);
@@ -3801,9 +3803,10 @@ BuildFieldValidBitmap(const GeneratedData& data, FieldId field_id) {
         if (fd.field_id() != field_id.get()) {
             continue;
         }
+        const auto& row_valid_data = GetFieldDataRowValidData(fd);
         for (int row = 0; row < row_count; ++row) {
             const bool valid =
-                fd.valid_data_size() == 0 ? true : fd.valid_data(row);
+                row_valid_data.empty() ? true : row_valid_data[row];
             if (valid) {
                 valid_bitmap[row >> 3] |= (1 << (row & 0x07));
             }
@@ -3852,7 +3855,7 @@ MakeNullableElementSearchWithNullAndEmptyRowsFixture() {
             target_values{7.0F, 7.0F, 7.0F, 7.0F, 1.0F, 0.0F, 0.0F, 0.0F};
         target_row->Add(target_values.begin(), target_values.end());
 
-        auto* valid_data = fd->mutable_valid_data();
+        auto* valid_data = fd->mutable_vectors()->mutable_valid_data();
         valid_data->Clear();
         valid_data->Add(false);  // row 0: null
         valid_data->Add(true);   // row 1: empty
@@ -3895,7 +3898,7 @@ MakeNullableElementSearchWithOnlyNullRowsFixture() {
                 ->Clear();
         }
 
-        auto* valid_data = fd->mutable_valid_data();
+        auto* valid_data = fd->mutable_vectors()->mutable_valid_data();
         valid_data->Clear();
         for (int row = 0; row < kNullableElemN; ++row) {
             valid_data->Add(false);
@@ -4664,10 +4667,11 @@ TEST(ElementFilterGrowingNullable, SearchAndSubscriptAcrossPhysicalChunks) {
         for (int d = 0; d < dim; ++d) {
             target->mutable_float_vector()->add_data(1.0f + d);
         }
-        field_data->mutable_valid_data()->Clear();
-        field_data->add_valid_data(false);
-        field_data->add_valid_data(true);
-        field_data->add_valid_data(true);
+        auto* valid_data = field_data->mutable_vectors()->mutable_valid_data();
+        valid_data->Clear();
+        valid_data->Add(false);
+        valid_data->Add(true);
+        valid_data->Add(true);
         break;
     }
 
@@ -4731,10 +4735,11 @@ TEST(ElementFilterGrowingNullable, SearchAndSubscriptAcrossPhysicalChunks) {
     auto data_array =
         segment->bulk_subscript(/*op_ctx=*/nullptr, vec_fid, seg_offsets, N);
     ASSERT_NE(data_array, nullptr);
-    ASSERT_EQ(data_array->valid_data_size(), N);
-    EXPECT_FALSE(data_array->valid_data(0));
-    EXPECT_TRUE(data_array->valid_data(1));
-    EXPECT_TRUE(data_array->valid_data(2));
+    const auto& valid_data = GetFieldDataRowValidData(*data_array);
+    ASSERT_EQ(valid_data.size(), N);
+    EXPECT_FALSE(valid_data[0]);
+    EXPECT_TRUE(valid_data[1]);
+    EXPECT_TRUE(valid_data[2]);
     const auto& out_rows = data_array->vectors().vector_array().data();
     ASSERT_EQ(out_rows.size(), N);
     EXPECT_EQ(out_rows[1].float_vector().data_size(), 0);

internal/core/unittest/test_external_take.cpp

@@ -501,10 +501,11 @@ AssertNullableDenseVectorTake(DataType data_type, int64_t dim) {
     ASSERT_EQ(results->fields_data_size(), 1);
 
     const auto& retrieved = results->fields_data(0);
-    ASSERT_EQ(retrieved.valid_data_size(), 3);
-    EXPECT_TRUE(retrieved.valid_data(0));
-    EXPECT_FALSE(retrieved.valid_data(1));
-    EXPECT_TRUE(retrieved.valid_data(2));
+    const auto& retrieved_valid_data = GetFieldDataRowValidData(retrieved);
+    ASSERT_EQ(retrieved_valid_data.size(), 3);
+    EXPECT_TRUE(retrieved_valid_data[0]);
+    EXPECT_FALSE(retrieved_valid_data[1]);
+    EXPECT_TRUE(retrieved_valid_data[2]);
     ASSERT_EQ(retrieved.vectors().dim(), dim);
 
     std::string actual;
@@ -534,10 +535,11 @@ AssertNullableDenseVectorTake(DataType data_type, int64_t dim) {
         search_plan.get(), offsets.data(), offsets.size(), search_results);
     ASSERT_TRUE(ok);
     auto& searched = search_results.output_fields_data_.at(info.vec_id);
-    ASSERT_EQ(searched->valid_data_size(), 3);
-    EXPECT_TRUE(searched->valid_data(0));
-    EXPECT_FALSE(searched->valid_data(1));
-    EXPECT_TRUE(searched->valid_data(2));
+    const auto& searched_valid_data = GetFieldDataRowValidData(*searched);
+    ASSERT_EQ(searched_valid_data.size(), 3);
+    EXPECT_TRUE(searched_valid_data[0]);
+    EXPECT_FALSE(searched_valid_data[1]);
+    EXPECT_TRUE(searched_valid_data[2]);
     ASSERT_EQ(searched->vectors().dim(), dim);
 }
 
@@ -564,10 +566,11 @@ AssertNullableSparseVectorTake() {
     ASSERT_EQ(results->fields_data_size(), 1);
 
     const auto& retrieved = results->fields_data(0);
-    ASSERT_EQ(retrieved.valid_data_size(), 3);
-    EXPECT_TRUE(retrieved.valid_data(0));
-    EXPECT_FALSE(retrieved.valid_data(1));
-    EXPECT_TRUE(retrieved.valid_data(2));
+    const auto& retrieved_valid_data = GetFieldDataRowValidData(retrieved);
+    ASSERT_EQ(retrieved_valid_data.size(), 3);
+    EXPECT_TRUE(retrieved_valid_data[0]);
+    EXPECT_FALSE(retrieved_valid_data[1]);
+    EXPECT_TRUE(retrieved_valid_data[2]);
     const auto& sparse = retrieved.vectors().sparse_float_vector();
     ASSERT_EQ(sparse.contents_size(), 2);
     EXPECT_EQ(sparse.contents(0), row0);
@@ -581,10 +584,11 @@ AssertNullableSparseVectorTake() {
         search_plan.get(), offsets.data(), offsets.size(), search_results);
     ASSERT_TRUE(ok);
     auto& searched = search_results.output_fields_data_.at(info.vec_id);
-    ASSERT_EQ(searched->valid_data_size(), 3);
-    EXPECT_TRUE(searched->valid_data(0));
-    EXPECT_FALSE(searched->valid_data(1));
-    EXPECT_TRUE(searched->valid_data(2));
+    const auto& searched_valid_data = GetFieldDataRowValidData(*searched);
+    ASSERT_EQ(searched_valid_data.size(), 3);
+    EXPECT_TRUE(searched_valid_data[0]);
+    EXPECT_FALSE(searched_valid_data[1]);
+    EXPECT_TRUE(searched_valid_data[2]);
     const auto& search_sparse = searched->vectors().sparse_float_vector();
     ASSERT_EQ(search_sparse.contents_size(), 2);
     EXPECT_EQ(search_sparse.contents(0), row0);
@@ -1332,10 +1336,11 @@ TEST(ExternalTakeTest, TryTakeForRetrieve_NullableVectorUsesCompactData) {
 
     auto& vec_data = results->fields_data(0);
     ASSERT_EQ(vec_data.field_id(), info.vec_id.get());
-    ASSERT_EQ(vec_data.valid_data_size(), 3);
-    EXPECT_TRUE(vec_data.valid_data(0));
-    EXPECT_FALSE(vec_data.valid_data(1));
-    EXPECT_TRUE(vec_data.valid_data(2));
+    const auto& valid_data = GetFieldDataRowValidData(vec_data);
+    ASSERT_EQ(valid_data.size(), 3);
+    EXPECT_TRUE(valid_data[0]);
+    EXPECT_FALSE(valid_data[1]);
+    EXPECT_TRUE(valid_data[2]);
 
     auto& fv = vec_data.vectors().float_vector();
     ASSERT_EQ(fv.data_size(), 2 * kVecDim);
@@ -1429,10 +1434,11 @@ TEST(ExternalTakeTest, TryTakeForRetrieve_NullableBinaryVectorUsesCompactData) {
 
     auto& binary_vec = results->fields_data(0);
     ASSERT_EQ(binary_vec.field_id(), info.binary_vec_id.get());
-    ASSERT_EQ(binary_vec.valid_data_size(), 3);
-    EXPECT_TRUE(binary_vec.valid_data(0));
-    EXPECT_FALSE(binary_vec.valid_data(1));
-    EXPECT_TRUE(binary_vec.valid_data(2));
+    const auto& valid_data = GetFieldDataRowValidData(binary_vec);
+    ASSERT_EQ(valid_data.size(), 3);
+    EXPECT_TRUE(valid_data[0]);
+    EXPECT_FALSE(valid_data[1]);
+    EXPECT_TRUE(valid_data[2]);
     EXPECT_EQ(binary_vec.vectors().binary_vector(),
               BuildDenseVectorBytesForRows({0, 2}, kBinaryVecDim / 8, 11));
 }
@@ -1565,10 +1571,11 @@ TEST(ExternalTakeTest, TryTakeForSearch_NullableVectorUsesCompactData) {
     ASSERT_TRUE(ok);
 
     auto& vec_data = results.output_fields_data_.at(info.vec_id);
-    ASSERT_EQ(vec_data->valid_data_size(), 3);
-    EXPECT_TRUE(vec_data->valid_data(0));
-    EXPECT_FALSE(vec_data->valid_data(1));
-    EXPECT_TRUE(vec_data->valid_data(2));
+    const auto& valid_data = GetFieldDataRowValidData(*vec_data);
+    ASSERT_EQ(valid_data.size(), 3);
+    EXPECT_TRUE(valid_data[0]);
+    EXPECT_FALSE(valid_data[1]);
+    EXPECT_TRUE(valid_data[2]);
 
     auto& fv = vec_data->vectors().float_vector();
     ASSERT_EQ(fv.data_size(), 2 * kVecDim);

internal/core/unittest/test_growing_concurrent_reopen.cpp

@@ -19,6 +19,7 @@
 
 #include "common/OpContext.h"
 #include "common/Schema.h"
+#include "common/Utils.h"
 #include "segcore/SegmentGrowingImpl.h"
 #include "test_utils/DataGen.h"
 
@@ -175,9 +176,10 @@ TEST(GrowingConcurrentReopenTest,
         auto evolved = seg_impl->bulk_subscript(
             nullptr, evolved_fid, offsets.data(), offsets.size());
         ASSERT_NE(evolved, nullptr);
-        ASSERT_EQ(evolved->valid_data_size(), N);
+        const auto& valid_data = GetFieldDataRowValidData(*evolved);
+        ASSERT_EQ(valid_data.size(), N);
         for (int i = 0; i < N; ++i) {
-            EXPECT_FALSE(evolved->valid_data(i));
+            EXPECT_FALSE(valid_data[i]);
         }
     }
 

internal/core/unittest/test_query.cpp

@@ -31,6 +31,7 @@
 #include "common/QueryResult.h"
 #include "common/Schema.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "common/VectorTrait.h"
 #include "common/protobuf_utils.h"
 #include "gtest/gtest.h"
@@ -684,7 +685,8 @@ TEST(Query, DISABLED_FillSegment) {
         auto output_i32_field_data =
             fields_data.at(i32_field_id)->scalars().int_data().data();
         ASSERT_EQ(output_i32_field_data.size(), topk * num_queries);
-        auto output_i32_valid_data = fields_data.at(i32_field_id)->valid_data();
+        auto output_i32_valid_data =
+            GetFieldDataRowValidData(*fields_data.at(i32_field_id));
         ASSERT_EQ(output_i32_valid_data.size(), topk * num_queries);
         auto float_field_id =
             schema->get_field_id(FieldName("lack_null_binlog"));
@@ -692,7 +694,7 @@ TEST(Query, DISABLED_FillSegment) {
             fields_data.at(float_field_id)->scalars().float_data().data();
         ASSERT_EQ(output_float_field_data.size(), topk * num_queries);
         auto output_float_valid_data =
-            fields_data.at(float_field_id)->valid_data();
+            GetFieldDataRowValidData(*fields_data.at(float_field_id));
         ASSERT_EQ(output_float_valid_data.size(), topk * num_queries);
         auto double_field_id =
             schema->get_field_id(FieldName("lack_default_value_binlog_double"));
@@ -700,7 +702,7 @@ TEST(Query, DISABLED_FillSegment) {
             fields_data.at(double_field_id)->scalars().double_data().data();
         ASSERT_EQ(output_double_field_data.size(), topk * num_queries);
         auto output_double_valid_data =
-            fields_data.at(double_field_id)->valid_data();
+            GetFieldDataRowValidData(*fields_data.at(double_field_id));
         ASSERT_EQ(output_double_valid_data.size(), topk * num_queries);
 
         auto bool_field_id =
@@ -709,7 +711,7 @@ TEST(Query, DISABLED_FillSegment) {
             fields_data.at(bool_field_id)->scalars().bool_data().data();
         ASSERT_EQ(output_bool_field_data.size(), topk * num_queries);
         auto output_bool_valid_data =
-            fields_data.at(bool_field_id)->valid_data();
+            GetFieldDataRowValidData(*fields_data.at(bool_field_id));
         ASSERT_EQ(output_bool_valid_data.size(), topk * num_queries);
 
         auto int_field_id =
@@ -717,7 +719,8 @@ TEST(Query, DISABLED_FillSegment) {
         auto output_int_field_data =
             fields_data.at(int_field_id)->scalars().int_data().data();
         ASSERT_EQ(output_int_field_data.size(), topk * num_queries);
-        auto output_int_valid_data = fields_data.at(int_field_id)->valid_data();
+        auto output_int_valid_data =
+            GetFieldDataRowValidData(*fields_data.at(int_field_id));
         ASSERT_EQ(output_int_valid_data.size(), topk * num_queries);
 
         auto int64_field_id =
@@ -726,7 +729,7 @@ TEST(Query, DISABLED_FillSegment) {
             fields_data.at(int64_field_id)->scalars().long_data().data();
         ASSERT_EQ(output_int64_field_data.size(), topk * num_queries);
         auto output_int64_valid_data =
-            fields_data.at(int64_field_id)->valid_data();
+            GetFieldDataRowValidData(*fields_data.at(int64_field_id));
         ASSERT_EQ(output_int64_valid_data.size(), topk * num_queries);
 
         auto float_field_id_default_value =
@@ -738,8 +741,8 @@ TEST(Query, DISABLED_FillSegment) {
                 .data();
         ASSERT_EQ(output_float_field_data_default_value.size(),
                   topk * num_queries);
-        auto output_float_valid_data_default_value =
-            fields_data.at(float_field_id_default_value)->valid_data();
+        auto output_float_valid_data_default_value = GetFieldDataRowValidData(
+            *fields_data.at(float_field_id_default_value));
         ASSERT_EQ(output_float_valid_data_default_value.size(),
                   topk * num_queries);
 
@@ -749,7 +752,7 @@ TEST(Query, DISABLED_FillSegment) {
             fields_data.at(varchar_field_id)->scalars().string_data().data();
         ASSERT_EQ(output_varchar_field_data.size(), topk * num_queries);
         auto output_varchar_valid_data =
-            fields_data.at(varchar_field_id)->valid_data();
+            GetFieldDataRowValidData(*fields_data.at(varchar_field_id));
         ASSERT_EQ(output_varchar_valid_data.size(), topk * num_queries);
 
         for (int i = 0; i < topk * num_queries; i++) {

internal/core/unittest/test_query_group_by.cpp

@@ -15,6 +15,7 @@
 #include <set>
 #include <unordered_map>
 #include <vector>
+#include "common/Utils.h"
 #include "test_utils/DataGen.h"
 #include "segcore/SegmentSealed.h"
 #include "plan/PlanNode.h"
@@ -927,7 +928,8 @@ TEST_P(QueryAggTest, RetrieveAggregationWithValidityBitmap) {
     auto& field_data = retrieve_results->fields_data(0);
 
     // Check that valid_data is properly populated for nullable fields
-    auto valid_data_size = field_data.valid_data_size();
+    const auto& valid_data = GetFieldDataRowValidData(field_data);
+    auto valid_data_size = valid_data.size();
     auto data_size = field_data.scalars().long_data().data_size();
 
     if (nullable) {
@@ -937,7 +939,7 @@ TEST_P(QueryAggTest, RetrieveAggregationWithValidityBitmap) {
         // Count valid and invalid entries
         int valid_count = 0;
         for (int i = 0; i < valid_data_size; i++) {
-            if (field_data.valid_data(i)) {
+            if (valid_data[i]) {
                 valid_count++;
             }
         }
@@ -949,7 +951,7 @@ TEST_P(QueryAggTest, RetrieveAggregationWithValidityBitmap) {
         // or all entries should be true if populated
         if (valid_data_size > 0) {
             for (int i = 0; i < valid_data_size; i++) {
-                EXPECT_TRUE(field_data.valid_data(i))
+                EXPECT_TRUE(valid_data[i])
                     << "All values should be valid for non-nullable field";
             }
         }

internal/core/unittest/test_schema_reopen.cpp

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include "common/Schema.h"
+#include "common/Utils.h"
 #include "segcore/SegmentGrowingImpl.h"
 #include "test_utils/DataGen.h"
 #include "test_utils/storage_test_utils.h"
@@ -92,9 +93,10 @@ TEST_F(SchemaReopenTest, LoadWithAbsentNullableVectorFieldShouldReadAllNull) {
     // row must read as null.
     seg_impl->FillAbsentFields();
     auto col = seg_impl->bulk_subscript(&op_ctx, added_fid, offsets.data(), N);
-    ASSERT_EQ(col->valid_data_size(), N);
+    const auto& loaded_valid_data = GetFieldDataRowValidData(*col);
+    ASSERT_EQ(loaded_valid_data.size(), N);
     for (int i = 0; i < N; ++i) {
-        ASSERT_FALSE(col->valid_data(i)) << "row " << i << " should be null";
+        ASSERT_FALSE(loaded_valid_data[i]) << "row " << i << " should be null";
     }
     ASSERT_EQ(col->vectors().float_vector().data_size(), 0);
 
@@ -114,9 +116,11 @@ TEST_F(SchemaReopenTest, LoadWithAbsentNullableVectorFieldShouldReadAllNull) {
     std::iota(all_offsets.begin(), all_offsets.end(), 0);
     col =
         seg_impl->bulk_subscript(&op_ctx, added_fid, all_offsets.data(), 2 * N);
-    ASSERT_EQ(col->valid_data_size(), 2 * N);
+    const auto& replayed_valid_data = GetFieldDataRowValidData(*col);
+    ASSERT_EQ(replayed_valid_data.size(), 2 * N);
     for (int i = 0; i < 2 * N; ++i) {
-        ASSERT_FALSE(col->valid_data(i)) << "row " << i << " should be null";
+        ASSERT_FALSE(replayed_valid_data[i])
+            << "row " << i << " should be null";
     }
 }
 

internal/core/unittest/test_scorer.cpp

@@ -504,7 +504,7 @@ TEST(BoostScoreRunnerTest, NativeFilterGivesNullRowsNoBoost) {
     // Every row satisfies the filter on its data bits; odd rows are NULL.
     auto* age_col =
         age_field_data->mutable_scalars()->mutable_long_data()->mutable_data();
-    auto* valid_col = age_field_data->mutable_valid_data();
+    auto* valid_col = age_field_data->mutable_scalars()->mutable_valid_data();
     ASSERT_EQ(valid_col->size(), N);
     for (int64_t i = 0; i < N; i++) {
         age_col->at(i) = i;

internal/core/unittest/test_sealed.cpp

@@ -48,6 +48,7 @@
 #include "common/QueryInfo.h"
 #include "common/QueryResult.h"
 #include "common/Schema.h"
+#include "common/Utils.h"
 #include "common/Span.h"
 #include "common/Types.h"
 #include "common/VectorArray.h"
@@ -2505,21 +2506,21 @@ TEST(Sealed, QueryAllFields) {
     EXPECT_EQ(float_array_result->scalars().array_data().data_size(),
               dataset_size);
 
-    EXPECT_EQ(bool_result->valid_data_size(), 0);
-    EXPECT_EQ(int8_result->valid_data_size(), 0);
-    EXPECT_EQ(int16_result->valid_data_size(), 0);
-    EXPECT_EQ(int32_result->valid_data_size(), 0);
-    EXPECT_EQ(int64_result->valid_data_size(), 0);
-    EXPECT_EQ(float_result->valid_data_size(), 0);
-    EXPECT_EQ(double_result->valid_data_size(), 0);
-    EXPECT_EQ(varchar_result->valid_data_size(), 0);
-    EXPECT_EQ(json_result->valid_data_size(), 0);
-    EXPECT_EQ(int_array_result->valid_data_size(), 0);
-    EXPECT_EQ(long_array_result->valid_data_size(), 0);
-    EXPECT_EQ(bool_array_result->valid_data_size(), 0);
-    EXPECT_EQ(string_array_result->valid_data_size(), 0);
-    EXPECT_EQ(double_array_result->valid_data_size(), 0);
-    EXPECT_EQ(float_array_result->valid_data_size(), 0);
+    EXPECT_TRUE(GetFieldDataRowValidData(*bool_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*int8_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*int16_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*int32_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*int64_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*float_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*double_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*varchar_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*json_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*int_array_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*long_array_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*bool_array_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*string_array_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*double_array_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*float_array_result).empty());
 }
 
 TEST(Sealed, QueryAllNullableFields) {
@@ -2676,21 +2677,26 @@ TEST(Sealed, QueryAllNullableFields) {
     EXPECT_EQ(float_array_result->scalars().array_data().data_size(),
               dataset_size);
 
-    EXPECT_EQ(bool_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(int8_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(int16_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(int32_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(float_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(double_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(varchar_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(json_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(geometry_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(int_array_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(long_array_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(bool_array_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(string_array_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(double_array_result->valid_data_size(), dataset_size);
-    EXPECT_EQ(float_array_result->valid_data_size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*bool_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*int8_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*int16_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*int32_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*float_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*double_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*varchar_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*json_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*geometry_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*int_array_result).size(), dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*long_array_result).size(),
+              dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*bool_array_result).size(),
+              dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*string_array_result).size(),
+              dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*double_array_result).size(),
+              dataset_size);
+    EXPECT_EQ(GetFieldDataRowValidData(*float_array_result).size(),
+              dataset_size);
 }
 
 using VectorArrayTestParam =
@@ -2799,8 +2805,8 @@ TEST_P(SealedVectorArrayTest, QueryVectorArrayAllFields) {
         VerifyVectorResults(result_vec, expected_vec, element_type);
     }
 
-    EXPECT_EQ(int64_result->valid_data_size(), 0);
-    EXPECT_EQ(array_vector_result->valid_data_size(), 0);
+    EXPECT_TRUE(GetFieldDataRowValidData(*int64_result).empty());
+    EXPECT_TRUE(GetFieldDataRowValidData(*array_vector_result).empty());
 }
 
 TEST_P(SealedVectorArrayTest, SearchVectorArray) {
@@ -3265,9 +3271,10 @@ TEST(SealedVectorArrayNullable, BulkSubscriptEmptyThenSingleVectorArrayRows) {
     auto result = sealed->bulk_subscript(
         nullptr, array_vec, offsets.data(), offsets.size());
     ASSERT_NE(result, nullptr);
-    ASSERT_EQ(result->valid_data_size(), row_count);
-    EXPECT_TRUE(result->valid_data(0));
-    EXPECT_TRUE(result->valid_data(1));
+    const auto& valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(valid_data.size(), row_count);
+    EXPECT_TRUE(valid_data[0]);
+    EXPECT_TRUE(valid_data[1]);
 
     const auto& rows = result->vectors().vector_array().data();
     ASSERT_EQ(rows.size(), row_count);
@@ -3400,13 +3407,14 @@ TEST(SealedVectorArrayNullable,
         nullptr, array_vec, offsets.data(), offsets.size());
 
     ASSERT_NE(result, nullptr);
-    ASSERT_EQ(result->valid_data_size(), offsets.size());
+    const auto& valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(valid_data.size(), offsets.size());
     ASSERT_EQ(result->vectors().vector_array().data_size(), offsets.size());
 
     for (size_t i = 0; i < offsets.size(); ++i) {
         auto logical_offset = offsets[i];
         auto expected_valid = logical_offset % 3 != 0;
-        EXPECT_EQ(result->valid_data(i), expected_valid);
+        EXPECT_EQ(valid_data[i], expected_valid);
         const auto& result_vec = result->vectors().vector_array().data(i);
         if (!expected_valid) {
             EXPECT_TRUE(result_vec.has_float_vector());
@@ -3531,10 +3539,11 @@ TEST(SealedVectorArrayNullable,
         nullptr, array_vec, offsets.data(), offsets.size());
 
     ASSERT_NE(result, nullptr);
-    ASSERT_EQ(result->valid_data_size(), offsets.size());
+    const auto& valid_data = GetFieldDataRowValidData(*result);
+    ASSERT_EQ(valid_data.size(), offsets.size());
     ASSERT_EQ(result->vectors().vector_array().data_size(), offsets.size());
-    for (int i = 0; i < result->valid_data_size(); ++i) {
-        EXPECT_FALSE(result->valid_data(i));
+    for (int i = 0; i < valid_data.size(); ++i) {
+        EXPECT_FALSE(valid_data[i]);
         EXPECT_EQ(
             result->vectors().vector_array().data(i).float_vector().data_size(),
             0);

internal/core/unittest/test_string_expr.cpp

@@ -1647,7 +1647,7 @@ TEST(AlwaysTrueStringPlan, QueryWithOutputFields) {
     ASSERT_EQ(retrieved->fields_data().size(), 1);
     ASSERT_EQ(retrieved->fields_data(0).scalars().string_data().data().size(),
               N);
-    ASSERT_EQ(retrieved->fields_data(0).valid_data_size(), 0);
+    ASSERT_TRUE(GetFieldDataRowValidData(retrieved->fields_data(0)).empty());
 }
 
 TEST(AlwaysTrueStringPlan, QueryWithOutputFieldsNullable) {
@@ -1689,7 +1689,8 @@ TEST(AlwaysTrueStringPlan, QueryWithOutputFieldsNullable) {
     ASSERT_EQ(retrieved->fields_data().size(), 1);
     ASSERT_EQ(retrieved->fields_data(0).scalars().string_data().data().size(),
               N / 2);
-    ASSERT_EQ(retrieved->fields_data(0).valid_data().size(), N / 2);
+    ASSERT_EQ(GetFieldDataRowValidData(retrieved->fields_data(0)).size(),
+              N / 2);
 }
 
 // Test: NOT (a IS NOT NULL) - verifies that the result valid bits are always true

internal/core/unittest/test_utils/DataGen.h

@@ -28,6 +28,7 @@
 #include "common/EasyAssert.h"
 #include "common/Schema.h"
 #include "common/Types.h"
+#include "common/Utils.h"
 #include "index/ScalarIndexSort.h"
 #include "index/VectorMemIndex.h"
 #include "segcore/Collection.h"
@@ -287,7 +288,8 @@ struct GeneratedData {
                 auto& field_meta = schema_->operator[](field_id);
                 Assert(field_meta.is_nullable());
                 FixedVector<bool> ret(raw_->num_rows());
-                auto src_data = target_field_data.valid_data().data();
+                auto src_data =
+                    GetFieldDataRowValidData(target_field_data).data();
                 std::copy_n(src_data, raw_->num_rows(), ret.data());
                 return ret;
             }
@@ -1435,6 +1437,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
     FieldDataPtr field_data = nullptr;
 
     auto element_type = field_meta.get_element_type();
+    const auto& row_valid_data = GetFieldDataRowValidData(*data);
     auto createFieldData =
         [&field_data, &raw_count, &element_type](
             const void* raw_data, DataType data_type, int64_t dim) {
@@ -1522,7 +1525,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
             case DataType::BOOL: {
                 auto raw_data = data->scalars().bool_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::BOOL, dim);
                 } else {
@@ -1535,7 +1538,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                 std::vector<int8_t> data_raw(src_data.size());
                 std::copy_n(src_data.data(), src_data.size(), data_raw.data());
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         data_raw.data(), raw_valid_data, DataType::INT8, dim);
                 } else {
@@ -1548,7 +1551,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                 std::vector<int16_t> data_raw(src_data.size());
                 std::copy_n(src_data.data(), src_data.size(), data_raw.data());
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         data_raw.data(), raw_valid_data, DataType::INT16, dim);
                 } else {
@@ -1559,7 +1562,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
             case DataType::INT32: {
                 auto raw_data = data->scalars().int_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::INT32, dim);
                 } else {
@@ -1570,7 +1573,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
             case DataType::INT64: {
                 auto raw_data = data->scalars().long_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::INT64, dim);
                 } else {
@@ -1581,7 +1584,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
             case DataType::FLOAT: {
                 auto raw_data = data->scalars().float_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::FLOAT, dim);
                 } else {
@@ -1592,7 +1595,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
             case DataType::DOUBLE: {
                 auto raw_data = data->scalars().double_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::DOUBLE, dim);
                 } else {
@@ -1604,7 +1607,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                 auto raw_data =
                     data->scalars().timestamptz_data().data().data();
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         raw_data, raw_valid_data, DataType::TIMESTAMPTZ, dim);
                 } else {
@@ -1617,7 +1620,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                 auto end = data->scalars().string_data().data().end();
                 std::vector<std::string> data_raw(begin, end);
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(data_raw.data(),
                                             raw_valid_data,
                                             DataType::VARCHAR,
@@ -1635,7 +1638,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                     data_raw[i] = Json(simdjson::padded_string(str));
                 }
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         data_raw.data(), raw_valid_data, DataType::JSON, dim);
                 } else {
@@ -1651,7 +1654,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                     data_raw[i] = std::string(str);
                 }
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(data_raw.data(),
                                             raw_valid_data,
                                             DataType::GEOMETRY,
@@ -1668,7 +1671,7 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
                     data_raw[i] = Array(src_data.at(i));
                 }
                 if (field_meta.is_nullable()) {
-                    auto raw_valid_data = data->valid_data().data();
+                    auto raw_valid_data = row_valid_data.data();
                     createNullableFieldData(
                         data_raw.data(), raw_valid_data, DataType::ARRAY, dim);
                 } else {

internal/distributed/proxy/httpserver/utils.go

@@ -2286,13 +2286,13 @@ func buildStructArrayFieldDataInternal(structSchema *schemapb.StructArrayFieldSc
 				Type:      schemapb.DataType_Array,
 				FieldName: short,
 				FieldId:   sub.GetFieldID(),
-				ValidData: validData,
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
 						Data: &schemapb.ScalarField_ArrayData{ArrayData: arrayArray},
 					},
 				},
 			})
+			typeutil.SetFieldDataValidData(subFieldData[len(subFieldData)-1], validData)
 		case schemapb.DataType_ArrayOfVector:
 			dim, err := getDim(sub)
 			if err != nil {
@@ -2320,7 +2320,6 @@ func buildStructArrayFieldDataInternal(structSchema *schemapb.StructArrayFieldSc
 				Type:      schemapb.DataType_ArrayOfVector,
 				FieldName: short,
 				FieldId:   sub.GetFieldID(),
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{
 					Vectors: &schemapb.VectorField{
 						Dim: dim,
@@ -2330,6 +2329,7 @@ func buildStructArrayFieldDataInternal(structSchema *schemapb.StructArrayFieldSc
 					},
 				},
 			})
+			typeutil.SetFieldDataValidData(subFieldData[len(subFieldData)-1], validData)
 		default:
 			return nil, merr.WrapErrParameterInvalidMsg("unsupported struct sub-field data type: %s", sub.GetDataType())
 		}
@@ -2361,10 +2361,10 @@ func newStructArrayRowAccessor(fd *schemapb.FieldData, schema *schemapb.Collecti
 		return accessor, nil
 	}
 
-	expectedValidData := subs[0].GetValidData()
+	expectedValidData := typeutil.GetFieldDataValidData(subs[0])
 	accessor.subAccessors = make([]*fieldDataRowAccessor, 0, len(subs))
 	for _, sub := range subs {
-		if !slices.Equal(expectedValidData, sub.GetValidData()) {
+		if !slices.Equal(expectedValidData, typeutil.GetFieldDataValidData(sub)) {
 			return nil, merr.WrapErrServiceInternalMsg(
 				"struct array field %s sub-field %s has inconsistent valid data",
 				fd.GetFieldName(), sub.GetFieldName())
@@ -3205,7 +3205,7 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
 		default:
 			return nil, merr.WrapErrParameterInvalidMsg("the type(%v) of field(%v) is not supported, use other sdk please", colData.Type, name)
 		}
-		colData.ValidData = validDataMap[name]
+		typeutil.SetFieldDataValidData(colData, validDataMap[name])
 		columns = append(columns, colData)
 	}
 	if isDynamic {
@@ -3502,8 +3502,8 @@ func genDynamicFields(fields []string, list []*schemapb.FieldData) []string {
 }
 
 func fieldDataRowCount(fieldData *schemapb.FieldData) (int64, error) {
-	if len(fieldData.GetValidData()) > 0 {
-		return int64(len(fieldData.GetValidData())), nil
+	if validData := typeutil.GetFieldDataValidData(fieldData); len(validData) > 0 {
+		return int64(len(validData)), nil
 	}
 	return fieldDataValueCount(fieldData)
 }
@@ -3578,8 +3578,8 @@ func fieldDataValueCount(fieldData *schemapb.FieldData) (int64, error) {
 		if len(subs) == 0 {
 			return 0, nil
 		}
-		if len(subs[0].GetValidData()) > 0 {
-			return int64(len(subs[0].GetValidData())), nil
+		if validData := typeutil.GetFieldDataValidData(subs[0]); len(validData) > 0 {
+			return int64(len(validData)), nil
 		}
 		switch subs[0].GetType() {
 		case schemapb.DataType_Array:
@@ -3603,7 +3603,7 @@ type fieldDataRowAccessor struct {
 func newFieldDataRowAccessor(fieldData *schemapb.FieldData) (*fieldDataRowAccessor, error) {
 	accessor := &fieldDataRowAccessor{
 		fieldData: fieldData,
-		validData: fieldData.GetValidData(),
+		validData: typeutil.GetFieldDataValidData(fieldData),
 	}
 	if len(accessor.validData) == 0 {
 		return accessor, nil

internal/distributed/proxy/httpserver/utils_test.go

@@ -1363,7 +1363,7 @@ func TestTextFieldDMLConversion(t *testing.T) {
 			require.NotNil(t, textFieldData)
 			assert.Equal(t, schemapb.DataType_Text, textFieldData.GetType())
 			assert.Equal(t, []string{"short text", longText}, textFieldData.GetScalars().GetStringData().GetData())
-			assert.Equal(t, []bool{true, false, true}, textFieldData.GetValidData())
+			assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(textFieldData))
 		})
 	}
 }
@@ -1520,7 +1520,7 @@ func TestInsertWithNullableVectorFields(t *testing.T) {
 				}
 			}
 			assert.NotNil(t, vectorFieldData)
-			assert.Equal(t, []bool{false, true, false}, vectorFieldData.GetValidData())
+			assert.Equal(t, []bool{false, true, false}, typeutil.GetFieldDataValidData(vectorFieldData))
 			testcase.checkData(t, vectorFieldData)
 		})
 	}
@@ -1563,7 +1563,7 @@ func TestPartialUpdateWithNullableExplicitNull(t *testing.T) {
 		assert.NoError(t, err)
 		nullableField := getFieldDataByName(fieldsData, "nullable")
 		assert.NotNil(t, nullableField)
-		assert.Equal(t, []bool{false, false}, nullableField.GetValidData())
+		assert.Equal(t, []bool{false, false}, typeutil.GetFieldDataValidData(nullableField))
 		assert.Empty(t, nullableField.GetScalars().GetLongData().GetData())
 	})
 
@@ -1594,7 +1594,7 @@ func TestPartialUpdateWithNullableExplicitNull(t *testing.T) {
 		assert.NoError(t, err)
 		nullableField := getFieldDataByName(fieldsData, "nullable")
 		assert.NotNil(t, nullableField)
-		assert.Equal(t, []bool{false, true}, nullableField.GetValidData())
+		assert.Equal(t, []bool{false, true}, typeutil.GetFieldDataValidData(nullableField))
 		assert.Equal(t, []int64{20}, nullableField.GetScalars().GetLongData().GetData())
 	})
 
@@ -1677,7 +1677,7 @@ func TestPartialUpdateWithNullableExplicitNull(t *testing.T) {
 		assert.NoError(t, err)
 		vectorFieldData := getFieldDataByName(fieldsData, FieldBookIntro)
 		assert.NotNil(t, vectorFieldData)
-		assert.Equal(t, []bool{false, false}, vectorFieldData.GetValidData())
+		assert.Equal(t, []bool{false, false}, typeutil.GetFieldDataValidData(vectorFieldData))
 		assert.Empty(t, vectorFieldData.GetVectors().GetFloatVector().GetData())
 		assert.Equal(t, int64(2), vectorFieldData.GetVectors().GetDim())
 	})
@@ -2209,7 +2209,8 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 			FieldName: FieldBookIntro,
 			Field: &schemapb.FieldData_Vectors{
 				Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{true, false, true},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{
 							Data: []float32{0.1, 0.2, 0.3, 0.4},
@@ -2217,7 +2218,6 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					},
 				},
 			},
-			ValidData: []bool{true, false, true},
 		}
 
 		rows, err := buildQueryResp(0, []string{FieldBookIntro}, []*schemapb.FieldData{fieldData}, nil, nil, true, nil)
@@ -2234,13 +2234,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 			FieldName: FieldBookIntro,
 			Field: &schemapb.FieldData_Vectors{
 				Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{false, false},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{},
 					},
 				},
 			},
-			ValidData: []bool{false, false},
 		}
 
 		rows, err := buildQueryResp(0, []string{FieldBookIntro}, []*schemapb.FieldData{fieldData}, nil, nil, true, nil)
@@ -2256,7 +2256,8 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 			FieldName: FieldBookIntro,
 			Field: &schemapb.FieldData_Vectors{
 				Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{true, false, true},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{
 							Data: []float32{
@@ -2268,7 +2269,6 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					},
 				},
 			},
-			ValidData: []bool{true, false, true},
 		}
 
 		_, err := buildQueryResp(0, []string{FieldBookIntro}, []*schemapb.FieldData{fieldData}, nil, nil, true, nil)
@@ -2288,13 +2288,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					FieldName: FieldBookIntro,
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false},
+							Dim:       2,
 							Data: &schemapb.VectorField_FloatVector{
 								FloatVector: &schemapb.FloatArray{Data: []float32{0.1, 0.2, 0.3}},
 							},
 						},
 					},
-					ValidData: []bool{true, false},
 				},
 			},
 			{
@@ -2304,13 +2304,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					FieldName: FieldBookIntro,
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 16,
+							ValidData: []bool{true, false},
+							Dim:       16,
 							Data: &schemapb.VectorField_BinaryVector{
 								BinaryVector: []byte{0x01, 0x02, 0x03},
 							},
 						},
 					},
-					ValidData: []bool{true, false},
 				},
 			},
 			{
@@ -2320,13 +2320,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					FieldName: FieldBookIntro,
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false},
+							Dim:       2,
 							Data: &schemapb.VectorField_Float16Vector{
 								Float16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05},
 							},
 						},
 					},
-					ValidData: []bool{true, false},
 				},
 			},
 			{
@@ -2336,13 +2336,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					FieldName: FieldBookIntro,
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false},
+							Dim:       2,
 							Data: &schemapb.VectorField_Bfloat16Vector{
 								Bfloat16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05},
 							},
 						},
 					},
-					ValidData: []bool{true, false},
 				},
 			},
 			{
@@ -2352,13 +2352,13 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 					FieldName: FieldBookIntro,
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false},
+							Dim:       2,
 							Data: &schemapb.VectorField_Int8Vector{
 								Int8Vector: []byte{0x01, 0x02, 0x03},
 							},
 						},
 					},
-					ValidData: []bool{true, false},
 				},
 			},
 		}
@@ -2380,12 +2380,12 @@ func TestBuildQueryRespWithNullableCompactFields(t *testing.T) {
 			FieldName: FieldWordCount,
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
+					ValidData: []bool{false, true},
 					Data: &schemapb.ScalarField_LongData{
 						LongData: &schemapb.LongArray{Data: []int64{20}},
 					},
 				},
 			},
-			ValidData: []bool{false, true},
 		}
 
 		rows, err := buildQueryResp(0, []string{FieldWordCount}, []*schemapb.FieldData{fieldData}, nil, nil, true, nil)
@@ -2789,6 +2789,250 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
 	}
 }
 
+func newNullableFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.DataType) []*schemapb.FieldData {
+	fieldData1 := schemapb.FieldData{
+		Type:      schemapb.DataType_Bool,
+		FieldName: "field-bool",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_BoolData{
+					BoolData: &schemapb.BoolArray{
+						Data: []bool{true, true, true},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData1)
+
+	fieldData2 := schemapb.FieldData{
+		Type:      schemapb.DataType_Int8,
+		FieldName: "field-int8",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_IntData{
+					IntData: &schemapb.IntArray{
+						Data: []int32{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData2)
+
+	fieldData3 := schemapb.FieldData{
+		Type:      schemapb.DataType_Int16,
+		FieldName: "field-int16",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_IntData{
+					IntData: &schemapb.IntArray{
+						Data: []int32{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData3)
+
+	fieldData4 := schemapb.FieldData{
+		Type:      schemapb.DataType_Int32,
+		FieldName: "field-int32",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_IntData{
+					IntData: &schemapb.IntArray{
+						Data: []int32{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData4)
+
+	fieldData5 := schemapb.FieldData{
+		Type:      schemapb.DataType_Float,
+		FieldName: "field-float",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_FloatData{
+					FloatData: &schemapb.FloatArray{
+						Data: []float32{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData5)
+
+	fieldData6 := schemapb.FieldData{
+		Type:      schemapb.DataType_Double,
+		FieldName: "field-double",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_DoubleData{
+					DoubleData: &schemapb.DoubleArray{
+						Data: []float64{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData6)
+
+	fieldData7 := schemapb.FieldData{
+		Type:      schemapb.DataType_String,
+		FieldName: "field-string",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_StringData{
+					StringData: &schemapb.StringArray{
+						Data: []string{"0", "1", "2"},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData7)
+
+	fieldData8 := schemapb.FieldData{
+		Type:      schemapb.DataType_VarChar,
+		FieldName: "field-varchar",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_StringData{
+					StringData: &schemapb.StringArray{
+						Data: []string{"0", "1", "2"},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData8)
+
+	fieldData9 := schemapb.FieldData{
+		Type:      schemapb.DataType_JSON,
+		FieldName: "field-json",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_JsonData{
+					JsonData: &schemapb.JSONArray{
+						Data: [][]byte{[]byte(`{"XXX": 0}`), []byte(`{"XXX": 0}`), []byte(`{"XXX": 0}`)},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData9)
+
+	fieldData10 := schemapb.FieldData{
+		Type:      schemapb.DataType_Array,
+		FieldName: "field-array",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_ArrayData{
+					ArrayData: &schemapb.ArrayArray{
+						Data: []*schemapb.ScalarField{
+							{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
+							{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
+							{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
+						},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+
+	fieldData11 := schemapb.FieldData{
+		Type:      schemapb.DataType_Int64,
+		FieldName: "field-int64",
+		Field: &schemapb.FieldData_Scalars{
+			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_LongData{
+					LongData: &schemapb.LongArray{
+						Data: []int64{0, 1, 2},
+					},
+				},
+			},
+		},
+		IsDynamic: false,
+	}
+	fieldDatas = append(fieldDatas, &fieldData11)
+
+	switch firstFieldType {
+	case schemapb.DataType_None:
+		return fieldDatas
+	case schemapb.DataType_Bool:
+		return []*schemapb.FieldData{&fieldData1}
+	case schemapb.DataType_Int8:
+		return []*schemapb.FieldData{&fieldData2}
+	case schemapb.DataType_Int16:
+		return []*schemapb.FieldData{&fieldData3}
+	case schemapb.DataType_Int32:
+		return []*schemapb.FieldData{&fieldData4}
+	case schemapb.DataType_Float:
+		return []*schemapb.FieldData{&fieldData5}
+	case schemapb.DataType_Double:
+		return []*schemapb.FieldData{&fieldData6}
+	case schemapb.DataType_String:
+		return []*schemapb.FieldData{&fieldData7}
+	case schemapb.DataType_VarChar:
+		return []*schemapb.FieldData{&fieldData8}
+	case schemapb.DataType_BinaryVector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_FloatVector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_Float16Vector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_BFloat16Vector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_Int8Vector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_Array:
+		return []*schemapb.FieldData{&fieldData10}
+	case schemapb.DataType_JSON:
+		return []*schemapb.FieldData{&fieldData9}
+	case schemapb.DataType_SparseFloatVector:
+		vectorField := generateVectorFieldData(firstFieldType)
+		return []*schemapb.FieldData{&vectorField}
+	case schemapb.DataType_Int64:
+		return []*schemapb.FieldData{&fieldData11}
+	default:
+		return []*schemapb.FieldData{
+			{
+				FieldName: "wrong-field-type",
+				Type:      firstFieldType,
+			},
+		}
+	}
+}
+
 func newSearchResult(results []map[string]interface{}) []map[string]interface{} {
 	for i, result := range results {
 		result["field-bool"] = true
@@ -4227,7 +4471,7 @@ func TestAnyToColumnsNullableStructArray(t *testing.T) {
 	subs := structFD.GetStructArrays().GetFields()
 	require.Len(t, subs, 2)
 	for _, sub := range subs {
-		assert.Equal(t, []bool{true, false, true}, sub.GetValidData())
+		assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(sub))
 	}
 	assert.Len(t, subs[0].GetScalars().GetArrayData().GetData(), 2)
 	assert.Len(t, subs[1].GetVectors().GetVectorArray().GetData(), 2)
@@ -4258,7 +4502,7 @@ func TestAnyToColumnsNullableStructArrayAllNull(t *testing.T) {
 	subs := structFD.GetStructArrays().GetFields()
 	require.Len(t, subs, 2)
 	for _, sub := range subs {
-		assert.Equal(t, []bool{false, false}, sub.GetValidData())
+		assert.Equal(t, []bool{false, false}, typeutil.GetFieldDataValidData(sub))
 	}
 	assert.Empty(t, subs[0].GetScalars().GetArrayData().GetData())
 	assert.Empty(t, subs[1].GetVectors().GetVectorArray().GetData())
@@ -4341,7 +4585,7 @@ func TestBuildQueryRespNullableStructArrayDense(t *testing.T) {
 		structSchema, []structArrayRow{first, nullPlaceholder, third})
 	require.NoError(t, err)
 	for _, sub := range structFD.GetStructArrays().GetFields() {
-		sub.ValidData = []bool{true, false, true}
+		typeutil.SetFieldDataValidData(sub, []bool{true, false, true})
 	}
 
 	resp, err := buildQueryResp(0, []string{"my_struct"}, []*schemapb.FieldData{structFD}, nil, nil, true, schema)
@@ -4362,8 +4606,8 @@ func TestBuildQueryRespNullableStructArrayRejectsMismatchedValidData(t *testing.
 	require.NoError(t, err)
 	subs := structFD.GetStructArrays().GetFields()
 	require.Len(t, subs, 2)
-	subs[0].ValidData = []bool{true}
-	subs[1].ValidData = []bool{false}
+	typeutil.SetFieldDataValidData(subs[0], []bool{true})
+	typeutil.SetFieldDataValidData(subs[1], []bool{false})
 
 	_, err = buildQueryResp(0, []string{"my_struct"}, []*schemapb.FieldData{structFD}, nil, nil, true, schema)
 	require.Error(t, err)

internal/distributed/proxy/httpserver/wrap_request.go

@@ -224,12 +224,6 @@ func (f *FieldData) AsSchemapb() (*schemapb.FieldData, error) {
 			data = append(data, *value)
 			validData[i] = true
 		}
-		// This wrapper does not have the collection schema, so only emit ValidData
-		// when the payload actually contains null. Proxy fills an all-true bitmap
-		// for nullable fields without nulls and requires no bitmap for non-nullable fields.
-		if hasNull {
-			ret.ValidData = validData
-		}
 		ret.Field = &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
 				Data: &schemapb.ScalarField_StringData{
@@ -239,6 +233,12 @@ func (f *FieldData) AsSchemapb() (*schemapb.FieldData, error) {
 				},
 			},
 		}
+		// This wrapper does not have the collection schema, so only emit ValidData
+		// when the payload actually contains null. Proxy fills an all-true bitmap
+		// for nullable fields without nulls and requires no bitmap for non-nullable fields.
+		if hasNull {
+			typeutil.SetFieldDataValidData(&ret, validData)
+		}
 	case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32:
 		data := []int32{}
 		err := json.Unmarshal(raw, &data)

internal/distributed/proxy/httpserver/wrap_request_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/json"
 	"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 func TestFieldData_AsSchemapb(t *testing.T) {
@@ -60,7 +61,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, schemapb.DataType_Text, result.GetType())
 		assert.Equal(t, []string{"a", "b", "c"}, result.GetScalars().GetStringData().GetData())
-		assert.Empty(t, result.GetValidData())
+		assert.Empty(t, typeutil.GetFieldDataValidData(result))
 	})
 	t.Run("text_error", func(t *testing.T) {
 		fieldData := FieldData{
@@ -86,7 +87,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, schemapb.DataType_Text, result.GetType())
 		assert.Equal(t, []string{"a", ""}, result.GetScalars().GetStringData().GetData())
-		assert.Equal(t, []bool{true, false, true}, result.GetValidData())
+		assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(result))
 	})
 	t.Run("bool_ok", func(t *testing.T) {
 		fieldData := FieldData{

internal/proxy/aggregate_operators_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/agg"
 	"github.com/milvus-io/milvus/pkg/v3/proto/planpb"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 type AggregateOperatorsSuite struct {
@@ -802,6 +803,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 		Type: schemapb.DataType_Int64,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true, false}, // 2nd and 4th are null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{
 						Data: []int64{10, 20, 30, 40},
@@ -809,7 +811,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{true, false, true, false}, // 2nd and 4th are null
 	}
 	int64Accessor.SetVals(fieldData)
 	assert.False(t, int64Accessor.IsNullAt(0), "index 0 should be valid")
@@ -845,6 +846,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 		Type: schemapb.DataType_VarChar,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, true, false}, // 1st and 3rd are null
 				Data: &schemapb.ScalarField_StringData{
 					StringData: &schemapb.StringArray{
 						Data: []string{"a", "b", "c"},
@@ -852,7 +854,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{false, true, false}, // 1st and 3rd are null
 	}
 	stringAccessor.SetVals(stringFieldData)
 	assert.True(t, stringAccessor.IsNullAt(0), "index 0 should be null")
@@ -866,6 +867,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 		Type: schemapb.DataType_Double,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false},
 				Data: &schemapb.ScalarField_DoubleData{
 					DoubleData: &schemapb.DoubleArray{
 						Data: []float64{1.1, 2.2},
@@ -873,7 +875,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{true, false},
 	}
 	float64Accessor.SetVals(floatFieldData)
 	assert.False(t, float64Accessor.IsNullAt(0))
@@ -886,6 +887,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 		Type: schemapb.DataType_Bool,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, true, false},
 				Data: &schemapb.ScalarField_BoolData{
 					BoolData: &schemapb.BoolArray{
 						Data: []bool{true, false, true},
@@ -893,7 +895,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{true, true, false},
 	}
 	boolAccessor.SetVals(boolFieldData)
 	assert.False(t, boolAccessor.IsNullAt(0))
@@ -912,6 +913,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Hash_NullValue() {
 		Type: schemapb.DataType_Int64,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false}, // 2nd is null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{
 						Data: []int64{100, 200},
@@ -919,7 +921,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Hash_NullValue() {
 				},
 			},
 		},
-		ValidData: []bool{true, false}, // 2nd is null
 	}
 	int64Accessor.SetVals(fieldData)
 
@@ -935,6 +936,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Hash_NullValue() {
 		Type: schemapb.DataType_VarChar,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, true}, // 1st is null
 				Data: &schemapb.ScalarField_StringData{
 					StringData: &schemapb.StringArray{
 						Data: []string{"hello", "world"},
@@ -942,7 +944,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Hash_NullValue() {
 				},
 			},
 		},
-		ValidData: []bool{false, true}, // 1st is null
 	}
 	stringAccessor.SetVals(stringFieldData)
 
@@ -973,8 +974,8 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull() {
 	nullFv := agg.NewNullFieldValue()
 	err := agg.AssembleSingleValue(nullFv, int64FieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(int64FieldData.ValidData))
-	assert.False(t, int64FieldData.ValidData[0], "null should have ValidData=false")
+	assert.Equal(t, 1, len(typeutil.GetFieldDataValidData(int64FieldData)))
+	assert.False(t, typeutil.GetFieldDataValidData(int64FieldData)[0], "null should have ValidData=false")
 	assert.Equal(t, 1, len(int64FieldData.GetScalars().GetLongData().GetData()))
 	assert.Equal(t, int64(0), int64FieldData.GetScalars().GetLongData().GetData()[0], "null should have default value 0")
 
@@ -982,8 +983,8 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull() {
 	nonNullFv := agg.NewFieldValue(int64(42))
 	err = agg.AssembleSingleValue(nonNullFv, int64FieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(int64FieldData.ValidData))
-	assert.True(t, int64FieldData.ValidData[1], "non-null should have ValidData=true")
+	assert.Equal(t, 2, len(typeutil.GetFieldDataValidData(int64FieldData)))
+	assert.True(t, typeutil.GetFieldDataValidData(int64FieldData)[1], "non-null should have ValidData=true")
 	assert.Equal(t, int64(42), int64FieldData.GetScalars().GetLongData().GetData()[1])
 
 	// Test null String value
@@ -1002,8 +1003,8 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull() {
 	nullStrFv := agg.NewNullFieldValue()
 	err = agg.AssembleSingleValue(nullStrFv, stringFieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(stringFieldData.ValidData))
-	assert.False(t, stringFieldData.ValidData[0])
+	assert.Equal(t, 1, len(typeutil.GetFieldDataValidData(stringFieldData)))
+	assert.False(t, typeutil.GetFieldDataValidData(stringFieldData)[0])
 	assert.Equal(t, "", stringFieldData.GetScalars().GetStringData().GetData()[0], "null string should be empty")
 
 	// Test null Double value
@@ -1022,7 +1023,7 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull() {
 	nullDoubleFv := agg.NewNullFieldValue()
 	err = agg.AssembleSingleValue(nullDoubleFv, doubleFieldData)
 	assert.NoError(t, err)
-	assert.False(t, doubleFieldData.ValidData[0])
+	assert.False(t, typeutil.GetFieldDataValidData(doubleFieldData)[0])
 	assert.Equal(t, float64(0), doubleFieldData.GetScalars().GetDoubleData().GetData()[0])
 
 	// Test null Bool value
@@ -1041,7 +1042,7 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull() {
 	nullBoolFv := agg.NewNullFieldValue()
 	err = agg.AssembleSingleValue(nullBoolFv, boolFieldData)
 	assert.NoError(t, err)
-	assert.False(t, boolFieldData.ValidData[0])
+	assert.False(t, typeutil.GetFieldDataValidData(boolFieldData)[0])
 	assert.False(t, boolFieldData.GetScalars().GetBoolData().GetData()[0], "null bool should be false")
 }
 
@@ -1092,15 +1093,15 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleRow_WithMixedNulls() {
 	assert.NoError(t, err)
 
 	// Check first column (non-null int32)
-	assert.True(t, fieldDatas[0].ValidData[0])
+	assert.True(t, typeutil.GetFieldDataValidData(fieldDatas[0])[0])
 	assert.Equal(t, int32(1), fieldDatas[0].GetScalars().GetIntData().GetData()[0])
 
 	// Check second column (null int64)
-	assert.False(t, fieldDatas[1].ValidData[0])
+	assert.False(t, typeutil.GetFieldDataValidData(fieldDatas[1])[0])
 	assert.Equal(t, int64(0), fieldDatas[1].GetScalars().GetLongData().GetData()[0])
 
 	// Check third column (non-null int64)
-	assert.True(t, fieldDatas[2].ValidData[0])
+	assert.True(t, typeutil.GetFieldDataValidData(fieldDatas[2])[0])
 	assert.Equal(t, int64(100), fieldDatas[2].GetScalars().GetLongData().GetData()[0])
 }
 
@@ -1114,6 +1115,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Float32_IsNullAt() {
 		Type: schemapb.DataType_Float,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true}, // 2nd is null
 				Data: &schemapb.ScalarField_FloatData{
 					FloatData: &schemapb.FloatArray{
 						Data: []float32{1.1, 2.2, 3.3},
@@ -1121,7 +1123,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Float32_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{true, false, true}, // 2nd is null
 	}
 	float32Accessor.SetVals(fieldData)
 	assert.False(t, float32Accessor.IsNullAt(0), "index 0 should be valid")
@@ -1156,6 +1157,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Timestamptz_IsNullAt() {
 		Type: schemapb.DataType_Timestamptz,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, true, false}, // 1st and 3rd are null
 				Data: &schemapb.ScalarField_TimestamptzData{
 					TimestamptzData: &schemapb.TimestamptzArray{
 						Data: []int64{1000, 2000, 3000},
@@ -1163,7 +1165,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Timestamptz_IsNullAt() {
 				},
 			},
 		},
-		ValidData: []bool{false, true, false}, // 1st and 3rd are null
 	}
 	tzAccessor.SetVals(fieldData)
 	assert.True(t, tzAccessor.IsNullAt(0), "index 0 should be null")
@@ -1198,6 +1199,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Float32_Hash_NullValue() {
 		Type: schemapb.DataType_Float,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false}, // 2nd is null
 				Data: &schemapb.ScalarField_FloatData{
 					FloatData: &schemapb.FloatArray{
 						Data: []float32{1.5, 2.5},
@@ -1205,7 +1207,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Float32_Hash_NullValue() {
 				},
 			},
 		},
-		ValidData: []bool{true, false}, // 2nd is null
 	}
 	float32Accessor.SetVals(fieldData)
 
@@ -1226,6 +1227,7 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Timestamptz_Hash_NullValue()
 		Type: schemapb.DataType_Timestamptz,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, true}, // 1st is null
 				Data: &schemapb.ScalarField_TimestamptzData{
 					TimestamptzData: &schemapb.TimestamptzArray{
 						Data: []int64{1000, 2000},
@@ -1233,7 +1235,6 @@ func (s *AggregateOperatorsSuite) TestFieldAccessor_Timestamptz_Hash_NullValue()
 				},
 			},
 		},
-		ValidData: []bool{false, true}, // 1st is null
 	}
 	tzAccessor.SetVals(fieldData)
 
@@ -1264,16 +1265,16 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull_Float32() {
 	nullFv := agg.NewNullFieldValue()
 	err := agg.AssembleSingleValue(nullFv, floatFieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(floatFieldData.ValidData))
-	assert.False(t, floatFieldData.ValidData[0], "null should have ValidData=false")
+	assert.Equal(t, 1, len(typeutil.GetFieldDataValidData(floatFieldData)))
+	assert.False(t, typeutil.GetFieldDataValidData(floatFieldData)[0], "null should have ValidData=false")
 	assert.Equal(t, float32(0), floatFieldData.GetScalars().GetFloatData().GetData()[0], "null should have default value 0")
 
 	// Test non-null Float32 value
 	nonNullFv := agg.NewFieldValue(float32(3.14))
 	err = agg.AssembleSingleValue(nonNullFv, floatFieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(floatFieldData.ValidData))
-	assert.True(t, floatFieldData.ValidData[1], "non-null should have ValidData=true")
+	assert.Equal(t, 2, len(typeutil.GetFieldDataValidData(floatFieldData)))
+	assert.True(t, typeutil.GetFieldDataValidData(floatFieldData)[1], "non-null should have ValidData=true")
 	assert.Equal(t, float32(3.14), floatFieldData.GetScalars().GetFloatData().GetData()[1])
 }
 
@@ -1297,16 +1298,16 @@ func (s *AggregateOperatorsSuite) TestAssembleSingleValue_WithNull_Timestamptz()
 	nullFv := agg.NewNullFieldValue()
 	err := agg.AssembleSingleValue(nullFv, tzFieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, len(tzFieldData.ValidData))
-	assert.False(t, tzFieldData.ValidData[0], "null should have ValidData=false")
+	assert.Equal(t, 1, len(typeutil.GetFieldDataValidData(tzFieldData)))
+	assert.False(t, typeutil.GetFieldDataValidData(tzFieldData)[0], "null should have ValidData=false")
 	assert.Equal(t, int64(0), tzFieldData.GetScalars().GetTimestamptzData().GetData()[0], "null should have default value 0")
 
 	// Test non-null Timestamptz value
 	nonNullFv := agg.NewFieldValue(int64(1234567890))
 	err = agg.AssembleSingleValue(nonNullFv, tzFieldData)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, len(tzFieldData.ValidData))
-	assert.True(t, tzFieldData.ValidData[1], "non-null should have ValidData=true")
+	assert.Equal(t, 2, len(typeutil.GetFieldDataValidData(tzFieldData)))
+	assert.True(t, typeutil.GetFieldDataValidData(tzFieldData)[1], "non-null should have ValidData=true")
 	assert.Equal(t, int64(1234567890), tzFieldData.GetScalars().GetTimestamptzData().GetData()[1])
 }
 

internal/proxy/highlighter.go

@@ -361,7 +361,7 @@ func rowAlignedStringFieldData(fieldData *schemapb.FieldData, rowNum int) ([]str
 	}
 
 	data := fieldData.GetScalars().GetStringData().GetData()
-	validData := fieldData.GetValidData()
+	validData := typeutil.GetFieldDataValidData(fieldData)
 	if len(validData) == 0 {
 		if len(data) != rowNum {
 			return nil, merr.WrapErrServiceInternalMsg(

internal/proxy/impl.go

@@ -2896,9 +2896,24 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest)
 	if err != nil {
 		rsp.Status = merr.Status(err)
 	}
+	projectSearchResultValidDataForLegacy(rsp)
 	return rsp, nil
 }
 
+func projectSearchResultValidDataForLegacy(result *milvuspb.SearchResults) {
+	resultData := result.GetResults()
+	if resultData == nil {
+		return
+	}
+	for _, field := range resultData.GetFieldsData() {
+		typeutil.ProjectFieldDataValidDataForLegacy(field)
+	}
+	for _, field := range resultData.GetGroupByFieldValues() {
+		typeutil.ProjectFieldDataValidDataForLegacy(field)
+	}
+	typeutil.ProjectFieldDataValidDataForLegacy(resultData.GetGroupByFieldValue())
+}
+
 func (node *Proxy) search(ctx context.Context, request *milvuspb.SearchRequest, optimizedSearch bool, isRecallEvaluation bool) (*milvuspb.SearchResults, bool, bool, bool, error) {
 	metrics.GetStats(ctx).
 		SetNodeID(paramtable.GetNodeID()).
@@ -3141,6 +3156,7 @@ func (node *Proxy) HybridSearch(ctx context.Context, request *milvuspb.HybridSea
 	if err2 != nil {
 		rsp.Status = merr.Status(err2)
 	}
+	projectSearchResultValidDataForLegacy(rsp)
 	return rsp, err
 }
 
@@ -3579,7 +3595,7 @@ func (node *Proxy) handleIfSearchByPK(ctx context.Context, request *milvuspb.Sea
 		PlaceholderGroup: placeholderBytes,
 	}
 
-	return fieldData.GetValidData(), nil
+	return typeutil.GetFieldDataValidData(fieldData), nil
 }
 
 // Flush notify data nodes to persist the data of collection.
@@ -3802,6 +3818,9 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (*
 	method := "Query"
 
 	res, storageCost, err := node.query(ctx, qt, sp)
+	for _, field := range res.GetFieldsData() {
+		typeutil.ProjectFieldDataValidDataForLegacy(field)
+	}
 
 	if Params.QueryNodeCfg.StorageUsageTrackingEnabled.GetAsBool() {
 		metrics.ProxyScannedRemoteMB.WithLabelValues(

internal/proxy/impl_test.go

@@ -65,6 +65,35 @@ import (
 	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
+func TestProjectSearchResultValidDataForLegacy(t *testing.T) {
+	validData := []bool{true, false}
+	newField := func(name string) *schemapb.FieldData {
+		return &schemapb.FieldData{
+			FieldName: name,
+			Field: &schemapb.FieldData_Scalars{
+				Scalars: &schemapb.ScalarField{ValidData: validData},
+			},
+		}
+	}
+	outputField := newField("output")
+	legacyGroupBy := newField("legacy_group_by")
+	groupBy := newField("group_by")
+	result := &milvuspb.SearchResults{
+		Results: &schemapb.SearchResultData{
+			FieldsData:         []*schemapb.FieldData{outputField},
+			GroupByFieldValue:  legacyGroupBy,
+			GroupByFieldValues: []*schemapb.FieldData{groupBy},
+		},
+	}
+
+	projectSearchResultValidDataForLegacy(result)
+
+	for _, field := range []*schemapb.FieldData{outputField, legacyGroupBy, groupBy} {
+		assert.Equal(t, validData, field.GetValidData())
+		assert.Equal(t, validData, field.GetScalars().GetValidData())
+	}
+}
+
 func TestProxy_InvalidateCollectionMetaCache_remove_stream(t *testing.T) {
 	paramtable.Init()
 	cache := globalMetaCache

internal/proxy/pipeline_trace.go

@@ -99,7 +99,7 @@ func (t *PipelineTrace) traceReduce(opName string, msg opMsg) {
 		if gbvs := rd.GetGroupByFieldValues(); len(gbvs) > 0 {
 			gbv := gbvs[0]
 			t.Set(prefix+".groupByRows", fieldDataLen(gbv))
-			t.Set(prefix+".groupByCards", scalarGroupByCards(gbv.GetScalars(), topks, gbv.GetValidData()))
+			t.Set(prefix+".groupByCards", scalarGroupByCards(gbv.GetScalars(), topks, typeutil.GetFieldDataValidData(gbv)))
 		}
 	}
 }
@@ -125,7 +125,7 @@ func (t *PipelineTrace) traceSearchResult(prefix string, msg opMsg, key string)
 	if gbvs := rd.GetGroupByFieldValues(); len(gbvs) > 0 {
 		gbv := gbvs[0]
 		t.Set(prefix+".groupByRows", fieldDataLen(gbv))
-		t.Set(prefix+".groupByCards", scalarGroupByCards(gbv.GetScalars(), topks, gbv.GetValidData()))
+		t.Set(prefix+".groupByCards", scalarGroupByCards(gbv.GetScalars(), topks, typeutil.GetFieldDataValidData(gbv)))
 	}
 }
 
@@ -184,7 +184,7 @@ func fieldDataLen(fd *schemapb.FieldData) int {
 	if fd == nil {
 		return 0
 	}
-	if vd := fd.GetValidData(); len(vd) > 0 {
+	if vd := typeutil.GetFieldDataValidData(fd); len(vd) > 0 {
 		return len(vd)
 	}
 	switch fd.GetField().(type) {

internal/proxy/pipeline_trace_test.go

@@ -410,9 +410,9 @@ func TestScalarGroupByCards_DoubleData(t *testing.T) {
 func TestFieldDataLen_WithValidData(t *testing.T) {
 	// ValidData length takes precedence over scalar array length
 	fd := &schemapb.FieldData{
-		ValidData: []bool{true, false, true},
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
 				Data: &schemapb.ScalarField_StringData{
 					StringData: &schemapb.StringArray{Data: []string{"a", "b"}}, // compact
 				},

internal/proxy/proxy_test.go

@@ -1508,7 +1508,7 @@ func TestProxy(t *testing.T) {
 			mask[i] = true
 		}
 		for _, sf := range mixedStruct.GetStructArrays().GetFields() {
-			sf.ValidData = mask
+			typeutil.SetFieldDataValidData(sf, mask)
 		}
 		mixedReq := &milvuspb.InsertRequest{
 			DbName:         dbName,
@@ -1878,7 +1878,7 @@ func TestProxy(t *testing.T) {
 			if fd.GetFieldName() != subI32Name && fd.GetFieldName() != subFVecName {
 				continue
 			}
-			for i, v := range fd.GetValidData() {
+			for i, v := range typeutil.GetFieldDataValidData(fd) {
 				assert.False(t, v, "row %d of sub-field %s should be null", i, fd.GetFieldName())
 			}
 		}
@@ -1906,7 +1906,7 @@ func TestProxy(t *testing.T) {
 				if fd.GetFieldName() != subI32Name && fd.GetFieldName() != subFVecName {
 					continue
 				}
-				for i, v := range fd.GetValidData() {
+				for i, v := range typeutil.GetFieldDataValidData(fd) {
 					assert.Equal(t, !c.expectNull, v,
 						"%s: sub-field %s row %d ValidData", c.label, fd.GetFieldName(), i)
 				}
@@ -2505,7 +2505,7 @@ func TestProxy(t *testing.T) {
 			if fd.GetFieldName() != subI32Name && fd.GetFieldName() != subFVecName {
 				continue
 			}
-			for i, v := range fd.GetValidData() {
+			for i, v := range typeutil.GetFieldDataValidData(fd) {
 				assert.False(t, v, "upserted row %d sub-field %s should be null", i, fd.GetFieldName())
 			}
 		}
@@ -2523,7 +2523,7 @@ func TestProxy(t *testing.T) {
 			mask[i] = true
 		}
 		for _, sf := range mixedStruct.GetStructArrays().GetFields() {
-			sf.ValidData = mask
+			typeutil.SetFieldDataValidData(sf, mask)
 		}
 		mixedUpsertReq := &milvuspb.UpsertRequest{
 			DbName:         dbName,

internal/proxy/search_agg/computer_test.go

@@ -427,11 +427,11 @@ func TestSearchAggregationComputerNullGrouping(t *testing.T) {
 		FieldsData: nil,
 		GroupByFieldValues: []*schemapb.FieldData{
 			{
-				FieldId:   101,
-				Type:      schemapb.DataType_Int64,
-				ValidData: []bool{false, false},
+				FieldId: 101,
+				Type:    schemapb.DataType_Int64,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{0, 0}}},
+					ValidData: []bool{false, false},
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{0, 0}}},
 				}},
 			},
 		},
@@ -1370,10 +1370,10 @@ func TestSearchAggregationComputerInterleavedNullGroupBy(t *testing.T) {
 			// Row sequence:  A | null | B | null | A
 			// ValidData:     t |  f   | t |  f   | t
 			{
-				FieldId:   101,
-				Type:      schemapb.DataType_VarChar,
-				ValidData: []bool{true, false, true, false, true},
+				FieldId: 101,
+				Type:    schemapb.DataType_VarChar,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
+					ValidData: []bool{true, false, true, false, true},
 					Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{
 						Data: []string{"A", "", "B", "", "A"},
 					}},
@@ -1421,7 +1421,7 @@ func testLongFieldData(fieldID int64, values []int64) *schemapb.FieldData {
 
 func testNullableLongFieldData(fieldID int64, values []int64, validData []bool) *schemapb.FieldData {
 	fd := testLongFieldData(fieldID, values)
-	fd.ValidData = validData
+	fd.GetScalars().ValidData = validData
 	return fd
 }
 

internal/proxy/search_pipeline.go

@@ -1528,7 +1528,6 @@ func (op *organizeOperator) emptyFieldDataAccordingFieldSchema(fieldData *schema
 		FieldName: fieldData.FieldName,
 		FieldId:   fieldData.FieldId,
 		IsDynamic: fieldData.IsDynamic,
-		ValidData: make([]bool, 0),
 	}
 	if fieldData.Type == schemapb.DataType_FloatVector ||
 		fieldData.Type == schemapb.DataType_BinaryVector ||
@@ -2219,7 +2218,7 @@ func compareNulls(validData []bool, i, j int, nullsFirst bool) (int, bool) {
 // It returns the final order comparison after applying ASC/DESC to non-null values.
 func compareOrderByField(field *schemapb.FieldData, orderBy OrderByField, idxI, idxJ int, cache jsonValueCache) (int, error) {
 	if orderBy.JSONPath != "" && field.GetType() == schemapb.DataType_JSON {
-		if cmp, handled := compareNulls(field.ValidData, idxI, idxJ, orderBy.NullsFirst); handled {
+		if cmp, handled := compareNulls(typeutil.GetFieldDataValidData(field), idxI, idxJ, orderBy.NullsFirst); handled {
 			return cmp, nil
 		}
 		valI := cache.getCachedJSONValue(orderBy.FieldName, orderBy.JSONPath, idxI)
@@ -2231,7 +2230,7 @@ func compareOrderByField(field *schemapb.FieldData, orderBy OrderByField, idxI,
 		return cmp, nil
 	}
 
-	if cmp, handled := compareNulls(field.ValidData, idxI, idxJ, orderBy.NullsFirst); handled {
+	if cmp, handled := compareNulls(typeutil.GetFieldDataValidData(field), idxI, idxJ, orderBy.NullsFirst); handled {
 		return cmp, nil
 	}
 	cmp, err := compareFieldDataAt(field, idxI, idxJ, orderBy.NullsFirst)
@@ -2424,7 +2423,7 @@ func isSameGroupByValue(field *schemapb.FieldData, i, j int) bool {
 // (see task_insert.go withNANCheck() -> validate_util.go -> typeutil.VerifyFloat).
 // Therefore, NaN values cannot exist in stored data and will never reach this comparison.
 func compareFieldDataAt(field *schemapb.FieldData, i, j int, nullsFirst bool) (int, error) {
-	if cmp, handled := compareNulls(field.ValidData, i, j, nullsFirst); handled {
+	if cmp, handled := compareNulls(typeutil.GetFieldDataValidData(field), i, j, nullsFirst); handled {
 		return cmp, nil
 	}
 
@@ -2621,7 +2620,7 @@ func (op *orderByOperator) reorderResults(result *milvuspb.SearchResults, indice
 }
 
 func prepareNullableFieldDataReorder(field *schemapb.FieldData, indices []int) ([]bool, []int, int, error) {
-	validData := field.GetValidData()
+	validData := typeutil.GetFieldDataValidData(field)
 	if len(validData) == 0 {
 		return nil, nil, 0, nil
 	}
@@ -2661,8 +2660,9 @@ func reorderNullableFloatVectorData(field *schemapb.FieldData, data []float32, w
 		return nil, merr.WrapErrServiceInternalMsg("reorderFieldData: nullable FloatVector field %s has %d elements, expected compact %d (valid=%d, dim=%d)", field.GetFieldName(), len(data), expected, validCount, width)
 	}
 	newData := make([]float32, 0, countValidRows(newValidData)*width)
+	validData := typeutil.GetFieldDataValidData(field)
 	for _, oldIdx := range indices {
-		if !field.ValidData[oldIdx] {
+		if !validData[oldIdx] {
 			continue
 		}
 		physicalIdx := logicalToPhysical[oldIdx]
@@ -2678,8 +2678,9 @@ func reorderNullableByteVectorData(field *schemapb.FieldData, typeName string, d
 		return nil, merr.WrapErrServiceInternalMsg("reorderFieldData: nullable %s field %s has %d bytes, expected compact %d (valid=%d, width=%d)", typeName, field.GetFieldName(), len(data), expected, validCount, width)
 	}
 	newData := make([]byte, 0, countValidRows(newValidData)*width)
+	validData := typeutil.GetFieldDataValidData(field)
 	for _, oldIdx := range indices {
-		if !field.ValidData[oldIdx] {
+		if !validData[oldIdx] {
 			continue
 		}
 		physicalIdx := logicalToPhysical[oldIdx]
@@ -2694,8 +2695,9 @@ func reorderNullableSparseVectorData(field *schemapb.FieldData, contents [][]byt
 		return nil, merr.WrapErrServiceInternalMsg("reorderFieldData: nullable SparseFloatVector field %s has %d elements, expected compact %d", field.GetFieldName(), len(contents), validCount)
 	}
 	newContents := make([][]byte, 0, countValidRows(newValidData))
+	validData := typeutil.GetFieldDataValidData(field)
 	for _, oldIdx := range indices {
-		if !field.ValidData[oldIdx] {
+		if !validData[oldIdx] {
 			continue
 		}
 		newContents = append(newContents, contents[logicalToPhysical[oldIdx]])
@@ -2992,7 +2994,7 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
 
 	// Reorder valid data if present
 	if newValidData != nil {
-		field.ValidData = newValidData
+		typeutil.SetFieldDataValidData(field, newValidData)
 	}
 	return nil
 }

internal/proxy/search_pipeline_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/milvus-io/milvus/pkg/v3/util/merr"
 	"github.com/milvus-io/milvus/pkg/v3/util/testutils"
 	"github.com/milvus-io/milvus/pkg/v3/util/timerecord"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 func TestSearchPipeline(t *testing.T) {
@@ -1003,10 +1004,10 @@ func (s *SearchPipelineSuite) TestHybridAssembleOpNullableVectorCompactData() {
 					Type:      schemapb.DataType_FloatVector,
 					FieldName: "nullable_vec",
 					FieldId:   101,
-					ValidData: []bool{false, true, true},
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{false, true, true},
+							Dim:       2,
 							Data: &schemapb.VectorField_FloatVector{
 								FloatVector: &schemapb.FloatArray{Data: []float32{20, 20, 30, 30}},
 							},
@@ -1032,10 +1033,10 @@ func (s *SearchPipelineSuite) TestHybridAssembleOpNullableVectorCompactData() {
 					Type:      schemapb.DataType_FloatVector,
 					FieldName: "nullable_vec",
 					FieldId:   101,
-					ValidData: []bool{true, false},
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false},
+							Dim:       2,
 							Data: &schemapb.VectorField_FloatVector{
 								FloatVector: &schemapb.FloatArray{Data: []float32{40, 40}},
 							},
@@ -1066,7 +1067,7 @@ func (s *SearchPipelineSuite) TestHybridAssembleOpNullableVectorCompactData() {
 	result := out[0].(*milvuspb.SearchResults).GetResults()
 	s.Require().Len(result.GetFieldsData(), 1)
 	field := result.GetFieldsData()[0]
-	s.Equal([]bool{true, false, false, true, true}, field.GetValidData())
+	s.Equal([]bool{true, false, false, true, true}, typeutil.GetFieldDataValidData(field))
 	s.Equal([]float32{30, 30, 40, 40, 20, 20}, field.GetVectors().GetFloatVector().GetData())
 }
 
@@ -1446,9 +1447,9 @@ func (s *SearchPipelineSuite) TestLexicalHighlightOpNullableStringKeepsEmptyHigh
 							FieldId:   100,
 							FieldName: testVarCharField,
 							Type:      schemapb.DataType_VarChar,
-							ValidData: tc.validData,
 							Field: &schemapb.FieldData_Scalars{
 								Scalars: &schemapb.ScalarField{
+									ValidData: tc.validData,
 									Data: &schemapb.ScalarField_StringData{
 										StringData: &schemapb.StringArray{Data: tc.stringData},
 									},
@@ -3449,12 +3450,12 @@ func (s *SearchPipelineSuite) TestCompareFieldDataAtWithNulls() {
 		FieldName: "nullable_field",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true}, // index 1 is null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
 				},
 			},
 		},
-		ValidData: []bool{true, false, true}, // index 1 is null
 	}
 
 	// NULLS FIRST
@@ -3474,12 +3475,12 @@ func (s *SearchPipelineSuite) TestCompareFieldDataAtWithNulls() {
 		FieldName: "nullable_field2",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, false, true}, // index 0 and 1 are null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
 				},
 			},
 		},
-		ValidData: []bool{false, false, true}, // index 0 and 1 are null
 	}
 	s.Equal(0, mustCompare(nullableField2, 0, 1)) // null == null
 
@@ -3816,10 +3817,10 @@ func (s *SearchPipelineSuite) TestOrderByOperatorReordersNullableVectorCompactOu
 				{
 					Type:      schemapb.DataType_FloatVector,
 					FieldName: "nullable_float_vec",
-					ValidData: []bool{true, false, true},
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 2,
+							ValidData: []bool{true, false, true},
+							Dim:       2,
 							Data: &schemapb.VectorField_FloatVector{
 								FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 5, 6}},
 							},
@@ -3829,10 +3830,10 @@ func (s *SearchPipelineSuite) TestOrderByOperatorReordersNullableVectorCompactOu
 				{
 					Type:      schemapb.DataType_SparseFloatVector,
 					FieldName: "nullable_sparse_vec",
-					ValidData: []bool{true, false, true},
 					Field: &schemapb.FieldData_Vectors{
 						Vectors: &schemapb.VectorField{
-							Dim: 3,
+							ValidData: []bool{true, false, true},
+							Dim:       3,
 							Data: &schemapb.VectorField_SparseFloatVector{
 								SparseFloatVector: &schemapb.SparseFloatArray{
 									Dim:      3,
@@ -3859,9 +3860,9 @@ func (s *SearchPipelineSuite) TestOrderByOperatorReordersNullableVectorCompactOu
 
 	s.Equal([]int64{2, 3, 1}, sortedResult.Results.Ids.GetIntId().Data)
 	s.Equal([]int64{10, 20, 30}, sortedResult.Results.FieldsData[0].GetScalars().GetLongData().Data)
-	s.Equal([]bool{false, true, true}, sortedResult.Results.FieldsData[1].GetValidData())
+	s.Equal([]bool{false, true, true}, typeutil.GetFieldDataValidData(sortedResult.Results.FieldsData[1]))
 	s.Equal([]float32{5, 6, 1, 2}, sortedResult.Results.FieldsData[1].GetVectors().GetFloatVector().GetData())
-	s.Equal([]bool{false, true, true}, sortedResult.Results.FieldsData[2].GetValidData())
+	s.Equal([]bool{false, true, true}, typeutil.GetFieldDataValidData(sortedResult.Results.FieldsData[2]))
 	s.Equal([][]byte{{0x03}, {0x01}}, sortedResult.Results.FieldsData[2].GetVectors().GetSparseFloatVector().GetContents())
 }
 
@@ -4417,9 +4418,9 @@ func (s *SearchPipelineSuite) TestOrderByOperatorNullableScalarNullOrdering() {
 					{
 						Type:      schemapb.DataType_Int64,
 						FieldName: "price",
-						ValidData: []bool{true, false, true, false},
 						Field: &schemapb.FieldData_Scalars{
 							Scalars: &schemapb.ScalarField{
+								ValidData: []bool{true, false, true, false},
 								Data: &schemapb.ScalarField_LongData{
 									LongData: &schemapb.LongArray{Data: []int64{30, 10, 40, 20}},
 								},
@@ -4475,7 +4476,7 @@ func (s *SearchPipelineSuite) TestOrderByOperatorNullableScalarNullOrdering() {
 			sortedResult := outputs[0].(*milvuspb.SearchResults)
 
 			s.Equal(tt.expectedIDs, sortedResult.Results.Ids.GetIntId().Data)
-			s.Equal(tt.expectedValid, sortedResult.Results.FieldsData[0].GetValidData())
+			s.Equal(tt.expectedValid, typeutil.GetFieldDataValidData(sortedResult.Results.FieldsData[0]))
 		})
 	}
 }
@@ -5520,12 +5521,12 @@ func (s *SearchPipelineSuite) TestCompareFieldDataAtNullable() {
 		FieldName: "nullable_field",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true}, // Index 1 is null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: []int64{100, 200, 300}},
 				},
 			},
 		},
-		ValidData: []bool{true, false, true}, // Index 1 is null
 	}
 
 	// NULLS FIRST
@@ -5546,12 +5547,12 @@ func (s *SearchPipelineSuite) TestCompareFieldDataAtNullable() {
 		FieldName: "nullable_field2",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{false, false, true}, // Index 0 and 1 are null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: []int64{100, 200, 300}},
 				},
 			},
 		},
-		ValidData: []bool{false, false, true}, // Index 0 and 1 are null
 	}
 	s.Equal(0, mustCompare(nullableField2, 0, 1)) // null == null
 }
@@ -6086,10 +6087,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_FloatVector,
 				FieldName: "nullable_float_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 5, 6}}},
+					ValidData: []bool{true, false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 5, 6}}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6101,10 +6102,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_BinaryVector,
 				FieldName: "nullable_binary_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  16,
-					Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02, 0x05, 0x06}},
+					ValidData: []bool{true, false, true},
+					Dim:       16,
+					Data:      &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02, 0x05, 0x06}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6116,10 +6117,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_Float16Vector,
 				FieldName: "nullable_float16_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x09, 0x0A, 0x0B, 0x0C}},
+					ValidData: []bool{true, false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x09, 0x0A, 0x0B, 0x0C}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6131,10 +6132,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_BFloat16Vector,
 				FieldName: "nullable_bfloat16_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x11, 0x12, 0x13, 0x14, 0x31, 0x32, 0x33, 0x34}},
+					ValidData: []bool{true, false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x11, 0x12, 0x13, 0x14, 0x31, 0x32, 0x33, 0x34}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6146,10 +6147,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_Int8Vector,
 				FieldName: "nullable_int8_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  4,
-					Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x09, 0x0A, 0x0B, 0x0C}},
+					ValidData: []bool{true, false, true},
+					Dim:       4,
+					Data:      &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x09, 0x0A, 0x0B, 0x0C}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6161,9 +6162,9 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_SparseFloatVector,
 				FieldName: "nullable_sparse_vec",
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: 3,
+					ValidData: []bool{true, false, true},
+					Dim:       3,
 					Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 						Dim:      3,
 						Contents: [][]byte{{0x01}, {0x03}},
@@ -6180,7 +6181,7 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorCompactData() {
 		s.Run(tt.name, func() {
 			err := reorderFieldData(tt.field, indices)
 			s.NoError(err)
-			s.Equal(expectedValidData, tt.field.GetValidData())
+			s.Equal(expectedValidData, typeutil.GetFieldDataValidData(tt.field))
 			tt.assert(tt.field)
 		})
 	}
@@ -6200,10 +6201,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_FloatVector,
 				FieldName: "nullable_float_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{}},
+					ValidData: []bool{false, false},
+					Dim:       2,
+					Data:      &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6215,10 +6216,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_BinaryVector,
 				FieldName: "nullable_binary_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  16,
-					Data: &schemapb.VectorField_BinaryVector{},
+					ValidData: []bool{false, false},
+					Dim:       16,
+					Data:      &schemapb.VectorField_BinaryVector{},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6230,10 +6231,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_Float16Vector,
 				FieldName: "nullable_float16_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Float16Vector{},
+					ValidData: []bool{false, false},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Float16Vector{},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6245,10 +6246,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_BFloat16Vector,
 				FieldName: "nullable_bfloat16_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Bfloat16Vector{},
+					ValidData: []bool{false, false},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Bfloat16Vector{},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6260,10 +6261,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_Int8Vector,
 				FieldName: "nullable_int8_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  4,
-					Data: &schemapb.VectorField_Int8Vector{},
+					ValidData: []bool{false, false},
+					Dim:       4,
+					Data:      &schemapb.VectorField_Int8Vector{},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6275,10 +6276,10 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 			field: &schemapb.FieldData{
 				Type:      schemapb.DataType_SparseFloatVector,
 				FieldName: "nullable_sparse_vec",
-				ValidData: []bool{false, false},
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  3,
-					Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{Dim: 3}},
+					ValidData: []bool{false, false},
+					Dim:       3,
+					Data:      &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{Dim: 3}},
 				}},
 			},
 			assert: func(field *schemapb.FieldData) {
@@ -6291,7 +6292,7 @@ func (s *SearchPipelineSuite) TestReorderFieldDataNullableVectorAllNullCompactDa
 		s.Run(tt.name, func() {
 			err := reorderFieldData(tt.field, indices)
 			s.NoError(err)
-			s.Equal(expectedValidData, tt.field.GetValidData())
+			s.Equal(expectedValidData, typeutil.GetFieldDataValidData(tt.field))
 			tt.assert(tt.field)
 		})
 	}
@@ -6304,12 +6305,12 @@ func (s *SearchPipelineSuite) TestReorderFieldDataWithValidData() {
 		FieldName: "nullable_field",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true}, // index 1 is null
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
 				},
 			},
 		},
-		ValidData: []bool{true, false, true}, // index 1 is null
 	}
 
 	// Reorder: [2, 0, 1]
@@ -6320,7 +6321,7 @@ func (s *SearchPipelineSuite) TestReorderFieldDataWithValidData() {
 	expectedData := []int64{30, 10, 20}
 	expectedValid := []bool{true, true, false} // null moves to index 2
 	s.Equal(expectedData, field.GetScalars().GetLongData().Data)
-	s.Equal(expectedValid, field.ValidData)
+	s.Equal(expectedValid, typeutil.GetFieldDataValidData(field))
 }
 
 // Test orderByOperator with Int32 field to cover reorderFieldData Int32 branch
@@ -6590,6 +6591,7 @@ func (s *SearchPipelineSuite) TestCompareOrderByFieldNullableJSON() {
 		FieldName: "metadata",
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true}, // Index 1 is null
 				Data: &schemapb.ScalarField_JsonData{
 					JsonData: &schemapb.JSONArray{
 						Data: [][]byte{
@@ -6601,7 +6603,6 @@ func (s *SearchPipelineSuite) TestCompareOrderByFieldNullableJSON() {
 				},
 			},
 		},
-		ValidData: []bool{true, false, true}, // Index 1 is null
 	}
 
 	fieldMap := map[string]*schemapb.FieldData{
@@ -7021,10 +7022,10 @@ func (s *SearchPipelineSuite) TestPickFieldDataWithNullableSparseVector() {
 			Type:      schemapb.DataType_SparseFloatVector,
 			FieldName: "sparse_vec",
 			FieldId:   101,
-			ValidData: []bool{true, false, true}, // Row 1 is null
 			Field: &schemapb.FieldData_Vectors{
 				Vectors: &schemapb.VectorField{
-					Dim: 700,
+					ValidData: []bool{true, false, true}, // Row 1 is null
+					Dim:       700,
 					Data: &schemapb.VectorField_SparseFloatVector{
 						SparseFloatVector: &schemapb.SparseFloatArray{
 							Dim:      700,
@@ -7049,7 +7050,7 @@ func (s *SearchPipelineSuite) TestPickFieldDataWithNullableSparseVector() {
 	// Verify sparse vector ValidData is reordered correctly
 	// Original ValidData: [true, false, true] for rows [1, 2, 3]
 	// After reorder to [3, 1, 2]: ValidData should be [true, true, false]
-	sparseValidData := result[1].GetValidData()
+	sparseValidData := typeutil.GetFieldDataValidData(result[1])
 	s.Equal([]bool{true, true, false}, sparseValidData)
 
 	// Verify sparse vector Contents has correct number of entries (2 non-null values)
@@ -7091,10 +7092,10 @@ func (s *SearchPipelineSuite) TestPickFieldDataWithAllNullSparseVector() {
 			Type:      schemapb.DataType_SparseFloatVector,
 			FieldName: "sparse_vec",
 			FieldId:   101,
-			ValidData: []bool{false, false}, // All null
 			Field: &schemapb.FieldData_Vectors{
 				Vectors: &schemapb.VectorField{
-					Dim: 700,
+					ValidData: []bool{false, false}, // All null
+					Dim:       700,
 					Data: &schemapb.VectorField_SparseFloatVector{
 						SparseFloatVector: &schemapb.SparseFloatArray{
 							Dim:      700,
@@ -7111,7 +7112,7 @@ func (s *SearchPipelineSuite) TestPickFieldDataWithAllNullSparseVector() {
 	s.NotNil(result)
 
 	// All should still be null after reorder
-	sparseValidData := result[1].GetValidData()
+	sparseValidData := typeutil.GetFieldDataValidData(result[1])
 	s.Equal([]bool{false, false}, sparseValidData)
 	s.Len(result[1].GetVectors().GetSparseFloatVector().GetContents(), 0)
 }
@@ -7172,6 +7173,6 @@ func (s *SearchPipelineSuite) TestPickFieldDataWithNullableSparseVectorMissingVa
 	result, err := pickFieldData(searchIDs, pkOffset, queryFields, schema, 12345)
 	s.NoError(err)
 	s.Equal([]int64{2, 1}, result[0].GetScalars().GetLongData().GetData())
-	s.Equal([]bool{false, false}, result[1].GetValidData())
+	s.Equal([]bool{false, false}, typeutil.GetFieldDataValidData(result[1]))
 	s.Empty(result[1].GetVectors().GetSparseFloatVector().GetContents())
 }

internal/proxy/search_reduce_util.go

@@ -443,9 +443,9 @@ func reduceAdvanceGroupBy(ctx context.Context, subSearchResultData []*schemapb.S
 			if gbv == nil {
 				continue
 			}
-			if gbv.GetValidData() != nil {
+			if validData := typeutil.GetFieldDataValidData(gbv); validData != nil {
 				fieldName := gbv.GetFieldName()
-				totalRows := len(gbv.GetValidData())
+				totalRows := len(validData)
 				gpFieldBuilder, err := typeutil.NewFieldDataBuilder(gbv.GetType(), true, totalRows)
 				if err != nil {
 					return nil, err

internal/proxy/search_reduce_util_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/milvus-io/milvus/internal/util/reduce"
 	"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
 	"github.com/milvus-io/milvus/pkg/v3/util/metric"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 type SearchReduceUtilTestSuite struct {
@@ -1058,9 +1059,9 @@ func (struts *SearchReduceUtilTestSuite) TestReduceAdvanceGroupBy_MultiFieldNull
 				FieldId:   101,
 				FieldName: "brand",
 				Type:      schemapb.DataType_VarChar,
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"A", "C"}}},
+					ValidData: []bool{true, false, true},
+					Data:      &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"A", "C"}}},
 				}},
 			},
 			multiGroupByTestStringField(102, []string{"X", "Y", "Z"}),
@@ -1074,7 +1075,7 @@ func (struts *SearchReduceUtilTestSuite) TestReduceAdvanceGroupBy_MultiFieldNull
 	struts.Require().Len(res.GetGroupByFieldValues(), 2)
 	nullableField := res.GetGroupByFieldValues()[0]
 	struts.Equal("brand", nullableField.GetFieldName())
-	struts.Equal([]bool{true, false, true}, nullableField.GetValidData())
+	struts.Equal([]bool{true, false, true}, typeutil.GetFieldDataValidData(nullableField))
 	struts.Equal([]string{"A", "", "C"}, nullableField.GetScalars().GetStringData().GetData())
 	struts.Equal([]string{"X", "Y", "Z"}, res.GetGroupByFieldValues()[1].GetScalars().GetStringData().GetData())
 }
@@ -1116,9 +1117,9 @@ func (struts *SearchReduceUtilTestSuite) TestReduceAdvanceGroupBy_MultiShardNull
 				FieldId:   101,
 				FieldName: "brand",
 				Type:      schemapb.DataType_VarChar,
-				ValidData: []bool{true, false},
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"A"}}},
+					ValidData: []bool{true, false},
+					Data:      &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"A"}}},
 				}},
 			},
 			multiGroupByTestStringField(102, []string{"X", "Y"}),
@@ -1135,9 +1136,9 @@ func (struts *SearchReduceUtilTestSuite) TestReduceAdvanceGroupBy_MultiShardNull
 				FieldId:   101,
 				FieldName: "brand",
 				Type:      schemapb.DataType_VarChar,
-				ValidData: []bool{false, true},
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"D"}}},
+					ValidData: []bool{false, true},
+					Data:      &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"D"}}},
 				}},
 			},
 			multiGroupByTestStringField(102, []string{"U", "V"}),
@@ -1152,7 +1153,7 @@ func (struts *SearchReduceUtilTestSuite) TestReduceAdvanceGroupBy_MultiShardNull
 	struts.Require().Len(res.GetGroupByFieldValues(), 2)
 	nullableField := res.GetGroupByFieldValues()[0]
 	struts.Equal("brand", nullableField.GetFieldName())
-	struts.Equal([]bool{true, false, false, true}, nullableField.GetValidData())
+	struts.Equal([]bool{true, false, false, true}, typeutil.GetFieldDataValidData(nullableField))
 	struts.Equal([]string{"A", "", "", "D"}, nullableField.GetScalars().GetStringData().GetData())
 	struts.Equal([]string{"X", "Y", "U", "V"}, res.GetGroupByFieldValues()[1].GetScalars().GetStringData().GetData())
 }

internal/proxy/struct_element_count_test.go

@@ -245,9 +245,9 @@ func TestCheckAndFlattenStructFieldDataAllowsNullableNullRowAndPresentRow(t *tes
 		},
 	}
 	field1 := structElementCountTestScalarArray("field1", []int32{1, 2})
-	field1.ValidData = []bool{false, true}
+	typeutil.SetFieldDataValidData(field1, []bool{false, true})
 	field2 := structElementCountTestVectorArray("field2", []float32{0.1, 0.2, 0.3, 0.4})
-	field2.ValidData = []bool{false, true}
+	typeutil.SetFieldDataValidData(field2, []bool{false, true})
 	insertMsg := structElementCountTestInsertMsg(structElementCountTestStructData(field1, field2))
 
 	err := checkAndFlattenStructFieldData(schema, insertMsg)

internal/proxy/task_insert.go

@@ -110,6 +110,9 @@ func (it *insertTask) PreExecute(ctx context.Context) error {
 		},
 		Timestamp: it.EndTs(),
 	}
+	if err := validateAndNormalizeFieldDataValidData(it.insertMsg.GetFieldsData()); err != nil {
+		return err
+	}
 
 	collectionName := it.insertMsg.CollectionName
 	log := mlog.With(mlog.String("collectionName", collectionName))

internal/proxy/task_search_test.go

@@ -2939,6 +2939,7 @@ func TestTaskSearch_reduceGroupBySearchResultData(t *testing.T) {
 			Type: schemapb.DataType_Int64,
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
+					ValidData: valids,
 					Data: &schemapb.ScalarField_LongData{
 						LongData: &schemapb.LongArray{
 							Data: groupByValues,
@@ -2946,7 +2947,6 @@ func TestTaskSearch_reduceGroupBySearchResultData(t *testing.T) {
 					},
 				},
 			},
-			ValidData: valids,
 		}
 		return result
 	}
@@ -3007,12 +3007,12 @@ func TestTaskSearch_reduceGroupBySearchResultData(t *testing.T) {
 				FieldId: 1,
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, true, true, true, false, true, true, true, true, false},
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{1, 2, 3, 4, 0, 1, 2, 3, 4, 0}},
 						},
 					},
 				},
-				ValidData: []bool{true, true, true, true, false, true, true, true, true, false},
 			},
 		},
 	}

internal/proxy/task_upsert.go

@@ -339,7 +339,7 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 				return err
 			}
 			for _, subField := range fieldData.GetStructArrays().GetFields() {
-				if len(subField.GetValidData()) != 0 && subFieldHasData(subField) {
+				if len(typeutil.GetFieldDataValidData(subField)) != 0 && subFieldHasData(subField) {
 					subFieldSchema, err := it.schema.SchemaHelper.GetFieldFromName(storedStructSubFieldName(structSchema.GetName(), subField.GetFieldName()))
 					if err != nil {
 						log.Info(ctx, "get struct sub-field schema failed", mlog.Err(err))
@@ -371,18 +371,18 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 		// For 2.5 collections where $meta is non-nullable with no default,
 		// ValidData must remain empty — CheckValidData expects len==0 for
 		// non-nullable fields.
-		if fieldData.GetIsDynamic() && len(fieldData.GetValidData()) == 0 &&
+		if fieldData.GetIsDynamic() && len(typeutil.GetFieldDataValidData(fieldData)) == 0 &&
 			(fieldSchema.GetNullable() || fieldSchema.GetDefaultValue() != nil) {
 			nRows := int(it.upsertMsg.InsertMsg.NRows())
 			validData := make([]bool, nRows)
 			for i := range validData {
 				validData[i] = true
 			}
-			fieldData.ValidData = validData
+			typeutil.SetFieldDataValidData(fieldData, validData)
 		}
 
 		// compatible with different nullable/default_value data format from sdk
-		if len(fieldData.GetValidData()) != 0 {
+		if len(typeutil.GetFieldDataValidData(fieldData)) != 0 {
 			var err error
 			if fieldSchema.GetDefaultValue() != nil {
 				err = FillWithDefaultValue(fieldData, fieldSchema, int(it.upsertMsg.InsertMsg.NRows()))
@@ -499,11 +499,12 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 			// Note: For fields containing default values, default values need to be set according to valid data during insertion,
 			// but query results fields do not set valid data when returning default value fields,
 			// therefore valid data needs to be manually set to true.
-			if fieldSchema.GetDefaultValue() != nil && len(existField.GetValidData()) == 0 {
-				existField.ValidData = make([]bool, existIDsLen)
-				for i := range existField.ValidData {
-					existField.ValidData[i] = true
+			if fieldSchema.GetDefaultValue() != nil && len(typeutil.GetFieldDataValidData(existField)) == 0 {
+				validData := make([]bool, existIDsLen)
+				for i := range validData {
+					validData[i] = true
 				}
+				typeutil.SetFieldDataValidData(existField, validData)
 			}
 
 			isNullableVector := typeutil.IsCompactNullableVectorFieldData(existField)
@@ -520,11 +521,12 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 				}
 				if isNullableVector {
 					// For nullable vector: only copy data for non-null rows
+					upsertValidData := typeutil.GetFieldDataValidData(upsertField)
 					upsertRowIndices := make([]int64, len(updateIdxInUpsert))
 					validDataIndices := make([]int64, 0, len(updateIdxInUpsert))
 					for i, upsertIdx := range updateIdxInUpsert {
 						upsertRowIndices[i] = int64(upsertIdx)
-						if upsertField.ValidData[upsertIdx] {
+						if upsertValidData[upsertIdx] {
 							validDataIndices = append(validDataIndices, upsertSrcIndices[i])
 						}
 					}
@@ -550,9 +552,10 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 				}
 			} else {
 				if isNullableVector {
+					existValidData := typeutil.GetFieldDataValidData(existField)
 					validDataIndices := make([]int64, 0, len(existIndices))
 					for i, existIdx := range existIndices {
-						if existField.ValidData[int(existIdx)] {
+						if existValidData[int(existIdx)] {
 							validDataIndices = append(validDataIndices, existSrcIndices[i])
 						}
 					}
@@ -627,12 +630,13 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 			}
 
 			if isNullableVector {
+				srcValidData := typeutil.GetFieldDataValidData(srcField)
 				rowIndices := make([]int64, len(insertIdxInUpsert))
 				validDataIndices := make([]int64, 0, len(insertIdxInUpsert))
 				for i, upsertIdx := range insertIdxInUpsert {
 					rowIndices[i] = int64(upsertIdx)
 					srcIdx := srcComputer.Compute(int64(upsertIdx))[0]
-					if srcField.ValidData[upsertIdx] {
+					if srcValidData[upsertIdx] {
 						validDataIndices = append(validDataIndices, srcIdx)
 					}
 				}
@@ -655,7 +659,7 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 			}
 			continue
 		}
-		if len(fieldData.GetValidData()) > 0 {
+		if len(typeutil.GetFieldDataValidData(fieldData)) > 0 {
 			err := ToCompressedFormatNullable(fieldData)
 			if err != nil {
 				log.Info(ctx, "convert to compressed format nullable failed", mlog.Err(err))
@@ -668,19 +672,20 @@ func (it *upsertTask) queryPreExecute(ctx context.Context) error {
 
 // ToCompressedFormatNullable converts nullable field data from full format to compressed format.
 func ToCompressedFormatNullable(field *schemapb.FieldData) error {
-	if getValidNumber(field.GetValidData()) == len(field.GetValidData()) {
+	validData := typeutil.GetFieldDataValidData(field)
+	if getValidNumber(validData) == len(validData) {
 		return nil
 	}
 	switch field.Field.(type) {
 	case *schemapb.FieldData_Scalars:
 		switch sd := field.GetScalars().GetData().(type) {
 		case *schemapb.ScalarField_BoolData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.BoolData.Data = make([]bool, 0)
 			} else {
 				ret := make([]bool, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.BoolData.Data[i])
 					}
@@ -689,12 +694,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_IntData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.IntData.Data = make([]int32, 0)
 			} else {
 				ret := make([]int32, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.IntData.Data[i])
 					}
@@ -703,12 +708,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_LongData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.LongData.Data = make([]int64, 0)
 			} else {
 				ret := make([]int64, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.LongData.Data[i])
 					}
@@ -717,12 +722,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_FloatData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.FloatData.Data = make([]float32, 0)
 			} else {
 				ret := make([]float32, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.FloatData.Data[i])
 					}
@@ -731,12 +736,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_DoubleData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.DoubleData.Data = make([]float64, 0)
 			} else {
 				ret := make([]float64, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.DoubleData.Data[i])
 					}
@@ -745,12 +750,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_StringData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.StringData.Data = make([]string, 0)
 			} else {
 				ret := make([]string, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.StringData.Data[i])
 					}
@@ -759,12 +764,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_JsonData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.JsonData.Data = make([][]byte, 0)
 			} else {
 				ret := make([][]byte, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.JsonData.Data[i])
 					}
@@ -773,12 +778,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_ArrayData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.ArrayData.Data = make([]*schemapb.ScalarField, 0)
 			} else {
 				ret := make([]*schemapb.ScalarField, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.ArrayData.Data[i])
 					}
@@ -786,12 +791,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 				sd.ArrayData.Data = ret
 			}
 		case *schemapb.ScalarField_TimestamptzData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.TimestamptzData.Data = make([]int64, 0)
 			} else {
 				ret := make([]int64, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.TimestamptzData.Data[i])
 					}
@@ -800,12 +805,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_GeometryWktData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.GeometryWktData.Data = make([]string, 0)
 			} else {
 				ret := make([]string, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.GeometryWktData.Data[i])
 					}
@@ -814,12 +819,12 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 			}
 
 		case *schemapb.ScalarField_GeometryData:
-			validRowNum := getValidNumber(field.GetValidData())
+			validRowNum := getValidNumber(validData)
 			if validRowNum == 0 {
 				sd.GeometryData.Data = make([][]byte, 0)
 			} else {
 				ret := make([][]byte, 0, validRowNum)
-				for i, valid := range field.GetValidData() {
+				for i, valid := range validData {
 					if valid {
 						ret = append(ret, sd.GeometryData.Data[i])
 					}
@@ -845,14 +850,18 @@ func ToCompressedFormatNullable(field *schemapb.FieldData) error {
 // GenNullableFieldData generates all-null nullable field data.
 // Scalar fields use expanded zero values; vector fields use compact empty data.
 func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schemapb.FieldData, error) {
+	withValidData := func(fieldData *schemapb.FieldData) *schemapb.FieldData {
+		typeutil.SetFieldDataValidData(fieldData, make([]bool, upsertIDSize))
+		return fieldData
+	}
+
 	switch field.DataType {
 	case schemapb.DataType_Bool:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_BoolData{
@@ -862,15 +871,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Int32:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_IntData{
@@ -880,15 +888,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Int64:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_LongData{
@@ -898,15 +905,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Float:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_FloatData{
@@ -916,15 +922,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Double:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_DoubleData{
@@ -934,15 +939,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_VarChar:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_StringData{
@@ -952,15 +956,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_JSON:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_JsonData{
@@ -970,15 +973,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Array:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_ArrayData{
@@ -989,15 +991,14 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Timestamptz:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_TimestamptzData{
@@ -1007,16 +1008,15 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	// the intput data of geometry field is in wkt format
 	case schemapb.DataType_Geometry:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
 			IsDynamic: field.IsDynamic,
-			ValidData: make([]bool, upsertIDSize),
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
 					Data: &schemapb.ScalarField_GeometryWktData{
@@ -1026,7 +1026,7 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 					},
 				},
 			},
-		}, nil
+		}), nil
 
 	// Nullable vector types
 	case schemapb.DataType_FloatVector:
@@ -1034,105 +1034,99 @@ func GenNullableFieldData(field *schemapb.FieldSchema, upsertIDSize int) (*schem
 		if err != nil {
 			return nil, err
 		}
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Dim:  dim,
 					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{}}},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Float16Vector:
 		dim, err := typeutil.GetDim(field)
 		if err != nil {
 			return nil, err
 		}
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Dim:  dim,
 					Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{}},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_BFloat16Vector:
 		dim, err := typeutil.GetDim(field)
 		if err != nil {
 			return nil, err
 		}
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Dim:  dim,
 					Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{}},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_BinaryVector:
 		dim, err := typeutil.GetDim(field)
 		if err != nil {
 			return nil, err
 		}
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Dim:  dim,
 					Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{}},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_SparseFloatVector:
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{},
 					}},
 				},
 			},
-		}, nil
+		}), nil
 
 	case schemapb.DataType_Int8Vector:
 		dim, err := typeutil.GetDim(field)
 		if err != nil {
 			return nil, err
 		}
-		return &schemapb.FieldData{
+		return withValidData(&schemapb.FieldData{
 			FieldId:   field.FieldID,
 			FieldName: field.Name,
 			Type:      field.DataType,
-			ValidData: make([]bool, upsertIDSize), // all false = all null
 			Field: &schemapb.FieldData_Vectors{
-				Vectors: &schemapb.VectorField{
+				Vectors: &schemapb.VectorField{ // all false = all null
 					Dim:  dim,
 					Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{}},
 				},
 			},
-		}, nil
+		}), nil
 
 	default:
 		return nil, merr.WrapErrParameterInvalidMsg("undefined data type:%s", field.DataType.String())
@@ -1148,14 +1142,25 @@ func GenNullableStructArrayFieldData(structField *schemapb.StructArrayFieldSchem
 				fieldName = originalName
 			}
 		}
-		fields = append(fields, &schemapb.FieldData{
+		fieldData := &schemapb.FieldData{
 			FieldName: fieldName,
 			FieldId:   subField.GetFieldID(),
 			Type:      subField.GetDataType(),
-			// The source field is indexed by original upsert row IDs before
-			// appending the insert subset, so ValidData uses the same row domain.
-			ValidData: make([]bool, rowCount),
-		})
+		}
+		// The source field is indexed by original upsert row IDs before
+		// appending the insert subset, so ValidData uses the same row domain.
+		validData := make([]bool, rowCount)
+		if typeutil.IsVectorType(subField.GetDataType()) {
+			fieldData.Field = &schemapb.FieldData_Vectors{
+				Vectors: &schemapb.VectorField{},
+			}
+		} else {
+			fieldData.Field = &schemapb.FieldData_Scalars{
+				Scalars: &schemapb.ScalarField{},
+			}
+		}
+		typeutil.SetFieldDataValidData(fieldData, validData)
+		fields = append(fields, fieldData)
 	}
 	return &schemapb.FieldData{
 		FieldId:   structField.GetFieldID(),
@@ -1174,11 +1179,11 @@ func ToCompressedFormatNullableStructField(fieldData *schemapb.FieldData) error
 		return nil
 	}
 	for _, subField := range structArrays.GetFields() {
-		if len(subField.GetValidData()) == 0 {
+		validData := typeutil.GetFieldDataValidData(subField)
+		if len(validData) == 0 {
 			continue
 		}
 
-		validData := subField.GetValidData()
 		validRows := getValidNumber(validData)
 		if validRows == 0 && !subFieldHasData(subField) {
 			continue
@@ -1235,6 +1240,10 @@ func validateWholeStructFieldDataForPartialUpdate(schemaHelper *typeutil.SchemaH
 
 	seenSubFieldIDs := make(map[int64]struct{}, len(structArrays.GetFields()))
 	for _, subField := range structArrays.GetFields() {
+		if !typeutil.ValidateAndNormalizeFieldDataValidData(subField) {
+			return merr.WrapErrParameterInvalidMsg("sub-field '%s' in struct '%s' has different legacy and field-specific valid_data",
+				subField.GetFieldName(), fieldData.GetFieldName())
+		}
 		subFieldSchema, err := schemaHelper.GetFieldFromName(storedStructSubFieldName(structSchema.GetName(), subField.GetFieldName()))
 		if err != nil {
 			return err
@@ -1248,9 +1257,10 @@ func validateWholeStructFieldDataForPartialUpdate(schemaHelper *typeutil.SchemaH
 			return merr.WrapErrParameterInvalidMsg("sub-field '%s' in struct '%s' expects type %s, got %s",
 				subField.GetFieldName(), fieldData.GetFieldName(), subFieldSchema.GetDataType().String(), subField.GetType().String())
 		}
-		if len(subField.GetValidData()) > 0 && len(subField.GetValidData()) != rowCount {
+		validData := typeutil.GetFieldDataValidData(subField)
+		if len(validData) > 0 && len(validData) != rowCount {
 			return merr.WrapErrParameterInvalidMsg("sub-field ValidData length mismatch in struct '%s': '%s' has %d, expected %d",
-				fieldData.GetFieldName(), subField.GetFieldName(), len(subField.GetValidData()), rowCount)
+				fieldData.GetFieldName(), subField.GetFieldName(), len(validData), rowCount)
 		}
 	}
 
@@ -1265,11 +1275,12 @@ func validateWholeStructFieldDataForPartialUpdate(schemaHelper *typeutil.SchemaH
 	// supported by this path.
 	if hasDataCount == 0 {
 		for _, subField := range structArrays.GetFields() {
-			if len(subField.GetValidData()) != 0 && len(subField.GetValidData()) != rowCount {
+			validData := typeutil.GetFieldDataValidData(subField)
+			if len(validData) != 0 && len(validData) != rowCount {
 				return merr.WrapErrParameterInvalidMsg("sub-field ValidData length mismatch in struct '%s': '%s' has %d, expected %d",
-					fieldData.GetFieldName(), subField.GetFieldName(), len(subField.GetValidData()), rowCount)
+					fieldData.GetFieldName(), subField.GetFieldName(), len(validData), rowCount)
 			}
-			for j, valid := range subField.GetValidData() {
+			for j, valid := range validData {
 				if valid {
 					return merr.WrapErrParameterInvalidMsg("sub-field '%s' in struct '%s' claims row %d is valid but no payload is provided",
 						subField.GetFieldName(), fieldData.GetFieldName(), j)
@@ -1287,22 +1298,23 @@ func validateWholeStructFieldDataForPartialUpdate(schemaHelper *typeutil.SchemaH
 	var refFieldName string
 	refInitialized := false
 	for _, subField := range structArrays.GetFields() {
+		validData := typeutil.GetFieldDataValidData(subField)
 		// Nullable is a property of the whole struct row. The sub-fields
 		// therefore must share the same null mask.
 		if structSchema.GetNullable() {
 			if !refInitialized {
-				refValidData = subField.GetValidData()
+				refValidData = validData
 				refFieldName = subField.GetFieldName()
 				refInitialized = true
 			} else {
-				if len(subField.GetValidData()) != len(refValidData) {
+				if len(validData) != len(refValidData) {
 					return merr.WrapErrParameterInvalidMsg("sub-field ValidData length mismatch in struct '%s': '%s' has %d, '%s' has %d",
-						fieldData.GetFieldName(), refFieldName, len(refValidData), subField.GetFieldName(), len(subField.GetValidData()))
+						fieldData.GetFieldName(), refFieldName, len(refValidData), subField.GetFieldName(), len(validData))
 				}
 				for j := range refValidData {
-					if subField.GetValidData()[j] != refValidData[j] {
+					if validData[j] != refValidData[j] {
 						return merr.WrapErrParameterInvalidMsg("sub-field ValidData mismatch in struct '%s' at row %d: '%s'=%v, '%s'=%v",
-							fieldData.GetFieldName(), j, refFieldName, refValidData[j], subField.GetFieldName(), subField.GetValidData()[j])
+							fieldData.GetFieldName(), j, refFieldName, refValidData[j], subField.GetFieldName(), validData[j])
 					}
 				}
 			}
@@ -1315,14 +1327,14 @@ func validateWholeStructFieldDataForPartialUpdate(schemaHelper *typeutil.SchemaH
 		if !ok {
 			return merr.WrapErrParameterInvalidMsg("struct sub-field '%s' has invalid field data", subField.GetFieldName())
 		}
-		if len(subField.GetValidData()) == 0 {
+		if len(validData) == 0 {
 			if payloadRows != rowCount {
 				return merr.WrapErrParameterInvalidMsg("struct sub-field '%s' payload row count %d does not match logical rows %d",
 					subField.GetFieldName(), payloadRows, rowCount)
 			}
 			continue
 		}
-		validRows := getValidNumber(subField.GetValidData())
+		validRows := getValidNumber(validData)
 		if payloadRows != validRows {
 			return merr.WrapErrParameterInvalidMsg("nullable struct sub-field '%s' payload must be compact: payload row count %d does not match valid rows %d",
 				subField.GetFieldName(), payloadRows, validRows)
@@ -1565,6 +1577,9 @@ func (it *upsertTask) PreExecute(ctx context.Context) error {
 		},
 		Timestamp: it.EndTs(),
 	}
+	if err := validateAndNormalizeFieldDataValidData(it.req.GetFieldsData()); err != nil {
+		return err
+	}
 
 	// check collection exists
 	collID, err := globalMetaCache.GetCollectionID(context.Background(), it.req.GetDbName(), collectionName)

internal/proxy/task_upsert_test.go

@@ -2808,15 +2808,15 @@ func TestUpdateTaskPreExecuteSnapshotsOriginalPartialFieldsBeforeMerge(t *testin
 	defer m.UnPatch()
 	schema := createTestSchema()
 	m = mockey.Mock((*MetaCache).GetCollectionInfo).Return(&collectionInfo{
-		updateTimestamp: 12345,
-		schema:          schema,
+		UpdateTimestamp: 12345,
+		Schema:          schema,
 	}, nil).Build()
 	defer m.UnPatch()
 	m = mockey.Mock((*MetaCache).GetCollectionSchema).Return(schema, nil).Build()
 	defer m.UnPatch()
 	m = mockey.Mock(isPartitionKeyMode).Return(false, nil).Build()
 	defer m.UnPatch()
-	m = mockey.Mock((*MetaCache).GetPartitionInfo).Return(&partitionInfo{name: "_default"}, nil).Build()
+	m = mockey.Mock((*MetaCache).GetPartitionInfo).Return(&partitionInfo{Name: "_default"}, nil).Build()
 	defer m.UnPatch()
 
 	fakeWAL := newPartialUpdateCASTestWAL(t, 9)
@@ -2825,7 +2825,7 @@ func TestUpdateTaskPreExecuteSnapshotsOriginalPartialFieldsBeforeMerge(t *testin
 	defer streaming.SetWALForTest(oldWAL)
 
 	m = mockey.Mock((*upsertTask).queryPreExecute).To(func(task *upsertTask, ctx context.Context) error {
-		task.req.FieldsData[1].ValidData = []bool{true, false, true}
+		typeutil.SetFieldDataValidData(task.req.FieldsData[1], []bool{true, false, true})
 		return nil
 	}).Build()
 	defer m.UnPatch()
@@ -2983,8 +2983,7 @@ func TestUpsertTask_queryPreExecute_MixLogic(t *testing.T) {
 			},
 			{
 				FieldName: "extra", FieldId: 102, Type: schemapb.DataType_VarChar,
-				Field:     &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"old1", "old2"}}}}},
-				ValidData: []bool{true, true},
+				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{ValidData: []bool{true, true}, Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"old1", "old2"}}}}},
 			},
 		},
 	}
@@ -3341,7 +3340,7 @@ func TestUpsertTask_queryPreExecute_StructWholeReplace(t *testing.T) {
 		}
 		field := profileField()
 		subField := field.GetStructArrays().GetFields()[0]
-		subField.ValidData = validData
+		typeutil.SetFieldDataValidData(subField, validData)
 		subField.GetScalars().GetArrayData().Data = rows
 		return field
 	}
@@ -3430,7 +3429,7 @@ func TestUpsertTask_queryPreExecute_StructWholeReplace(t *testing.T) {
 		profile := findProfile(task)
 		if assert.NotNil(t, profile) {
 			subField := profile.GetStructArrays().GetFields()[0]
-			assert.Equal(t, []bool{true, false}, subField.GetValidData())
+			assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(subField))
 			rows := subField.GetScalars().GetArrayData().GetData()
 			require.Len(t, rows, 1)
 			assert.Equal(t, []int32{111}, rows[0].GetIntData().GetData())
@@ -3883,6 +3882,7 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 			FieldName: "timestamp_field",
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
+					ValidData: []bool{true, false, true, false},
 					Data: &schemapb.ScalarField_TimestamptzData{
 						TimestamptzData: &schemapb.TimestamptzArray{
 							Data: []int64{1000, 0, 3000, 0},
@@ -3890,13 +3890,12 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 					},
 				},
 			},
-			ValidData: []bool{true, false, true, false},
 		}
 
 		err := ToCompressedFormatNullable(field)
 		assert.NoError(t, err)
 		assert.Equal(t, []int64{1000, 3000}, field.GetScalars().GetTimestamptzData().GetData())
-		assert.Equal(t, []bool{true, false, true, false}, field.ValidData)
+		assert.Equal(t, []bool{true, false, true, false}, typeutil.GetFieldDataValidData(field))
 	})
 
 	t.Run("geometry WKT with null values", func(t *testing.T) {
@@ -3905,6 +3904,7 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 			FieldName: "geometry_field",
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
+					ValidData: []bool{true, false, true},
 					Data: &schemapb.ScalarField_GeometryWktData{
 						GeometryWktData: &schemapb.GeometryWktArray{
 							Data: []string{"POINT (1 2)", "", "POINT (5 6)"},
@@ -3912,7 +3912,6 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 					},
 				},
 			},
-			ValidData: []bool{true, false, true},
 		}
 
 		err := ToCompressedFormatNullable(field)
@@ -3926,6 +3925,7 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 			FieldName: "geometry_field",
 			Field: &schemapb.FieldData_Scalars{
 				Scalars: &schemapb.ScalarField{
+					ValidData: []bool{true, false, true},
 					Data: &schemapb.ScalarField_GeometryData{
 						GeometryData: &schemapb.GeometryArray{
 							Data: [][]byte{{0x01, 0x02}, nil, {0x05, 0x06}},
@@ -3933,7 +3933,6 @@ func TestToCompressedFormatNullable_GeometryAndTimestamptz(t *testing.T) {
 					},
 				},
 			},
-			ValidData: []bool{true, false, true},
 		}
 
 		err := ToCompressedFormatNullable(field)
@@ -3959,7 +3958,7 @@ func TestGenNullableFieldData_GeometryAndTimestamptz(t *testing.T) {
 		assert.NotNil(t, fieldData)
 		assert.Equal(t, int64(100), fieldData.FieldId)
 		assert.Equal(t, "timestamp_field", fieldData.FieldName)
-		assert.Len(t, fieldData.ValidData, upsertIDSize)
+		assert.Len(t, typeutil.GetFieldDataValidData(fieldData), upsertIDSize)
 		assert.Len(t, fieldData.GetScalars().GetTimestamptzData().GetData(), upsertIDSize)
 	})
 
@@ -3978,7 +3977,7 @@ func TestGenNullableFieldData_GeometryAndTimestamptz(t *testing.T) {
 		assert.NotNil(t, fieldData)
 		assert.Equal(t, int64(101), fieldData.FieldId)
 		assert.Equal(t, "geometry_field", fieldData.FieldName)
-		assert.Len(t, fieldData.ValidData, upsertIDSize)
+		assert.Len(t, typeutil.GetFieldDataValidData(fieldData), upsertIDSize)
 		assert.Len(t, fieldData.GetScalars().GetGeometryWktData().GetData(), upsertIDSize)
 	})
 }
@@ -4675,8 +4674,8 @@ func TestUpsertTask_queryPreExecute_NullableFields(t *testing.T) {
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: vecData}}}},
 			},
 			{
-				FieldName: "nullable_vec", FieldId: 102, Type: schemapb.DataType_FloatVector, ValidData: validData,
-				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: nullableData}}}},
+				FieldName: "nullable_vec", FieldId: 102, Type: schemapb.DataType_FloatVector,
+				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{ValidData: validData, Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: nullableData}}}},
 			},
 		}
 	}
@@ -4725,8 +4724,8 @@ func TestUpsertTask_queryPreExecute_NullableFields(t *testing.T) {
 					Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: vecData}}}},
 				},
 				{
-					FieldName: "nullable_vec", FieldId: 102, Type: schemapb.DataType_FloatVector, ValidData: validData,
-					Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: nullableData}}}},
+					FieldName: "nullable_vec", FieldId: 102, Type: schemapb.DataType_FloatVector,
+					Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{ValidData: validData, Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: nullableData}}}},
 				},
 			},
 		}
@@ -4759,7 +4758,7 @@ func TestUpsertTask_queryPreExecute_NullableFields(t *testing.T) {
 	assert.Empty(t, task1a.deletePKs.GetIntId().GetData())
 	assert.Equal(t, []int64{1}, task1a.insertFieldData[0].GetScalars().GetLongData().GetData())
 	assert.Equal(t, []float32{1, 1, 1, 1}, task1a.insertFieldData[1].GetVectors().GetFloatVector().GetData())
-	assert.Equal(t, []bool{false}, task1a.insertFieldData[2].ValidData)
+	assert.Equal(t, []bool{false}, typeutil.GetFieldDataValidData(task1a.insertFieldData[2]))
 	assert.Empty(t, task1a.insertFieldData[2].GetVectors().GetFloatVector().GetData())
 
 	// Step 1b: Empty data, upsert pk2(all) -> insert, nullable_vec=[102,...]
@@ -4787,7 +4786,7 @@ func TestUpsertTask_queryPreExecute_NullableFields(t *testing.T) {
 	task3a := runUpsert(createPartialCols([]int64{3}), queryResult(nil))
 	assert.Empty(t, task3a.deletePKs.GetIntId().GetData())
 	assert.Equal(t, []int64{3}, task3a.insertFieldData[0].GetScalars().GetLongData().GetData())
-	assert.Equal(t, []bool{false}, task3a.insertFieldData[2].ValidData)
+	assert.Equal(t, []bool{false}, typeutil.GetFieldDataValidData(task3a.insertFieldData[2]))
 	assert.Empty(t, task3a.insertFieldData[2].GetVectors().GetFloatVector().GetData())
 
 	// Step 3b: Empty data, upsert pk4(all) -> insert, nullable_vec=[104,...]
@@ -4810,7 +4809,7 @@ func TestUpsertTask_queryPreExecute_NullableFields(t *testing.T) {
 	// Update rows pk3,pk4: nullable_vec from existing data (ValidData=true)
 	// Insert rows pk5,pk6: nullable_vec generated by GenNullableFieldData (null, ValidData=false)
 	// ValidData has 4 elements, FloatVector only contains data for ValidData=true rows
-	assert.Equal(t, []bool{true, true, false, false}, task4b.insertFieldData[2].ValidData)
+	assert.Equal(t, []bool{true, true, false, false}, typeutil.GetFieldDataValidData(task4b.insertFieldData[2]))
 	assert.Equal(t, []float32{303, 303, 303, 303, 304, 304, 304, 304}, task4b.insertFieldData[2].GetVectors().GetFloatVector().GetData())
 }
 
@@ -4848,9 +4847,9 @@ func TestUpsertTask_GenNullableFieldData(t *testing.T) {
 				assert.Equal(t, field.FieldID, result.FieldId)
 				assert.Equal(t, field.Name, result.FieldName)
 				assert.Equal(t, tc.dataType, result.Type)
-				assert.Equal(t, upsertIDSize, len(result.ValidData))
+				assert.Equal(t, upsertIDSize, len(typeutil.GetFieldDataValidData(result)))
 				// All ValidData should be false (null)
-				for _, v := range result.ValidData {
+				for _, v := range typeutil.GetFieldDataValidData(result) {
 					assert.False(t, v)
 				}
 			})
@@ -4884,9 +4883,9 @@ func TestUpsertTask_GenNullableFieldData(t *testing.T) {
 				assert.Equal(t, field.FieldID, result.FieldId)
 				assert.Equal(t, field.Name, result.FieldName)
 				assert.Equal(t, tc.dataType, result.Type)
-				assert.Equal(t, upsertIDSize, len(result.ValidData))
+				assert.Equal(t, upsertIDSize, len(typeutil.GetFieldDataValidData(result)))
 				// All ValidData should be false (null)
-				for _, v := range result.ValidData {
+				for _, v := range typeutil.GetFieldDataValidData(result) {
 					assert.False(t, v)
 				}
 				assert.NotNil(t, result.GetVectors())
@@ -4904,7 +4903,7 @@ func TestUpsertTask_GenNullableFieldData(t *testing.T) {
 		result, err := GenNullableFieldData(field, upsertIDSize)
 		assert.NoError(t, err)
 		assert.NotNil(t, result)
-		assert.Equal(t, upsertIDSize, len(result.ValidData))
+		assert.Equal(t, upsertIDSize, len(typeutil.GetFieldDataValidData(result)))
 		assert.NotNil(t, result.GetVectors().GetSparseFloatVector())
 	})
 
@@ -4949,8 +4948,7 @@ func TestUpsertTask_queryPreExecute_DefaultValueWithValidData(t *testing.T) {
 		},
 		{
 			FieldName: "default_col", FieldId: 102, Type: schemapb.DataType_VarChar,
-			Field:     &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"a", "b"}}}}},
-			ValidData: []bool{true, true, false},
+			Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{ValidData: []bool{true, true, false}, Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"a", "b"}}}}},
 		},
 	}
 
@@ -5039,8 +5037,7 @@ func TestUpsertTask_queryPreExecute_DefaultValueError(t *testing.T) {
 		{
 			// ValidData length (2) doesn't match numRows (3) → FillWithDefaultValue returns error
 			FieldName: "default_col", FieldId: 102, Type: schemapb.DataType_VarChar,
-			Field:     &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"a"}}}}},
-			ValidData: []bool{true, false},
+			Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{ValidData: []bool{true, false}, Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"a"}}}}},
 		},
 	}
 
@@ -5194,7 +5191,7 @@ func TestUpsertTask_queryPreExecute_DynamicFieldValidData(t *testing.T) {
 		metaData := metaField.GetScalars().GetJsonData().GetData()
 		assert.Equal(t, 3, len(metaData), "merged $meta should have 3 rows")
 		// ValidData should also have 3 entries (2 from update + 1 from insert)
-		assert.Equal(t, 3, len(metaField.GetValidData()), "ValidData length should match row count")
+		assert.Equal(t, 3, len(typeutil.GetFieldDataValidData(metaField)), "ValidData length should match row count")
 	})
 
 	t.Run("dynamic field without ValidData is auto-filled by queryPreExecute", func(t *testing.T) {
@@ -5280,7 +5277,7 @@ func TestUpsertTask_queryPreExecute_DynamicFieldValidData(t *testing.T) {
 		assert.NotNil(t, metaField)
 		metaData := metaField.GetScalars().GetJsonData().GetData()
 		assert.Equal(t, 3, len(metaData), "merged $meta should have 3 rows")
-		validData := metaField.GetValidData()
+		validData := typeutil.GetFieldDataValidData(metaField)
 		assert.Equal(t, 3, len(validData),
 			"queryPreExecute auto-fills ValidData, merge produces correct length 3")
 	})
@@ -5385,7 +5382,7 @@ func TestUpsertTask_queryPreExecute_DynamicFieldValidData(t *testing.T) {
 		metaData := metaField.GetScalars().GetJsonData().GetData()
 		assert.Equal(t, 3, len(metaData), "merged $meta should have 3 rows")
 		// For non-nullable $meta, ValidData should remain empty (not auto-filled)
-		assert.Empty(t, metaField.GetValidData(),
+		assert.Empty(t, typeutil.GetFieldDataValidData(metaField),
 			"non-nullable $meta should NOT have ValidData auto-filled")
 	})
 }

internal/proxy/util.go

@@ -1173,7 +1173,7 @@ func autoGenDynamicFieldData(schema *schemapb.CollectionSchema, data [][]byte) *
 			for i := range validData {
 				validData[i] = true
 			}
-			fd.ValidData = validData
+			typeutil.SetFieldDataValidData(fd, validData)
 			break
 		}
 	}
@@ -1216,6 +1216,21 @@ func validateFieldDataColumns(columns []*schemapb.FieldData, schema *schemaInfo)
 	return nil
 }
 
+// validateAndNormalizeFieldDataValidData validates compatibility fields once
+// when a user payload enters the proxy, then keeps only the current
+// field-specific representation for internal processing.
+func validateAndNormalizeFieldDataValidData(fields []*schemapb.FieldData) error {
+	for _, field := range fields {
+		if !typeutil.ValidateAndNormalizeFieldDataValidData(field) {
+			return merr.WrapErrParameterInvalidMsg(
+				"field %s has different legacy and field-specific valid_data",
+				field.GetFieldName(),
+			)
+		}
+	}
+	return nil
+}
+
 // fillFieldPropertiesOnly fills field properties (FieldId, Type, ElementType) from schema.
 // It assumes that columns have been validated and does not perform validation.
 // Use validateFieldDataColumns before calling this function if validation is needed.
@@ -1881,7 +1896,7 @@ func checkFieldsDataBySchema(ctx context.Context, allFields []*schemapb.FieldSch
 			if err != nil {
 				return err
 			}
-			dataToAppend.ValidData = make([]bool, insertMsg.GetNumRows())
+			typeutil.SetFieldDataValidData(dataToAppend, make([]bool, insertMsg.GetNumRows()))
 			insertMsg.FieldsData = append(insertMsg.FieldsData, dataToAppend)
 		}
 	}
@@ -1961,6 +1976,10 @@ func checkAndFlattenStructFieldData(schema *schemapb.CollectionSchema, insertMsg
 		// either have data or all be empty. Partial presence is invalid.
 		hasDataCount := 0
 		for _, subField := range structArrays.StructArrays.Fields {
+			if !typeutil.ValidateAndNormalizeFieldDataValidData(subField) {
+				return merr.WrapErrParameterInvalidMsg("sub-field '%s' in struct '%s' has different legacy and field-specific valid_data",
+					subField.GetFieldName(), structName)
+			}
 			if subFieldHasData(subField) {
 				hasDataCount++
 			}
@@ -1971,7 +1990,7 @@ func checkAndFlattenStructFieldData(schema *schemapb.CollectionSchema, insertMsg
 			// omitted entirely. Reject illegal ValidData first: when no payload is
 			// provided, any ValidData[i]==true contradicts itself.
 			for _, subField := range structArrays.StructArrays.Fields {
-				for j, v := range subField.ValidData {
+				for j, v := range typeutil.GetFieldDataValidData(subField) {
 					if v {
 						return merr.WrapErrParameterInvalidMsg("sub-field '%s' in struct '%s' claims row %d is valid but no payload is provided",
 							subField.FieldName, structName, j)
@@ -1995,20 +2014,21 @@ func checkAndFlattenStructFieldData(schema *schemapb.CollectionSchema, insertMsg
 			var refFieldName string
 			refInitialized := false
 			for _, subField := range structArrays.StructArrays.Fields {
+				validData := typeutil.GetFieldDataValidData(subField)
 				if !refInitialized {
-					refValidData = subField.ValidData
+					refValidData = validData
 					refFieldName = subField.FieldName
 					refInitialized = true
 					continue
 				}
-				if len(subField.ValidData) != len(refValidData) {
+				if len(validData) != len(refValidData) {
 					return merr.WrapErrParameterInvalidMsg("sub-field ValidData length mismatch in struct '%s': '%s' has %d, '%s' has %d",
-						structName, refFieldName, len(refValidData), subField.FieldName, len(subField.ValidData))
+						structName, refFieldName, len(refValidData), subField.FieldName, len(validData))
 				}
 				for j := range refValidData {
-					if subField.ValidData[j] != refValidData[j] {
+					if validData[j] != refValidData[j] {
 						return merr.WrapErrParameterInvalidMsg("sub-field ValidData mismatch in struct '%s' at row %d: '%s'=%v, '%s'=%v",
-							structName, j, refFieldName, refValidData[j], subField.FieldName, subField.ValidData[j])
+							structName, j, refFieldName, refValidData[j], subField.FieldName, validData[j])
 					}
 				}
 			}
@@ -2151,7 +2171,7 @@ func checkAndFlattenStructFieldData(schema *schemapb.CollectionSchema, insertMsg
 
 			if expectedArrayLen == -1 {
 				expectedArrayLen = currentArrayLen
-				firstValidData = subField.GetValidData()
+				firstValidData = typeutil.GetFieldDataValidData(subField)
 			} else if currentArrayLen != expectedArrayLen {
 				return merr.WrapErrParameterInvalidMsg("inconsistent array length in struct field '%s': expected %d, got %d for sub-field '%s'",
 					structName, expectedArrayLen, currentArrayLen, subField.FieldName)
@@ -2206,13 +2226,16 @@ func checkAndFlattenStructFieldData(schema *schemapb.CollectionSchema, insertMsg
 
 		for _, subField := range structArrays.StructArrays.Fields {
 			transformedFieldName := storedStructSubFieldName(structName, subField.FieldName)
+			validData := typeutil.GetFieldDataValidData(subField)
+			// Field is shared by the flattened copy. Normalize validity before
+			// sharing it so the source and flattened field cannot conflict.
+			typeutil.SetFieldDataValidData(subField, validData)
 			subFieldCopy := &schemapb.FieldData{
 				FieldName: transformedFieldName,
 				FieldId:   subField.FieldId,
 				Type:      subField.Type,
 				Field:     subField.Field,
 				IsDynamic: subField.IsDynamic,
-				ValidData: subField.ValidData,
 			}
 
 			flattenedFields = append(flattenedFields, subFieldCopy)

internal/proxy/util_test.go

@@ -6184,9 +6184,9 @@ func TestCheckAndFlattenStructFieldData_ValidDataCopied(t *testing.T) {
 									FieldName: subFieldName,
 									FieldId:   201,
 									Type:      schemapb.DataType_Array,
-									ValidData: validData,
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: validData,
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6215,7 +6215,8 @@ func TestCheckAndFlattenStructFieldData_ValidDataCopied(t *testing.T) {
 	for _, fd := range insertMsg.GetFieldsData() {
 		if fd.FieldName == transformedName {
 			found = true
-			assert.Equal(t, validData, fd.GetValidData(), "ValidData should be preserved in flattened sub-field")
+			assert.Equal(t, validData, typeutil.GetFieldDataValidData(fd), "ValidData should be preserved in flattened sub-field")
+			assert.Nil(t, fd.GetValidData(), "flattened sub-field should use only field-specific ValidData")
 			break
 		}
 	}
@@ -6379,9 +6380,9 @@ func TestCheckAndFlattenStructFieldData_RequiredMissingButNullablePresent(t *tes
 									FieldName: "sub_b",
 									FieldId:   301,
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{true, false},
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: []bool{true, false},
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6464,16 +6465,18 @@ func TestCheckAndFlattenStructFieldData_AllNullWithInitializedVectorsOneof(t *te
 									FieldName: "tag",
 									FieldId:   201,
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{false, false},
+									Field: &schemapb.FieldData_Scalars{
+										Scalars: &schemapb.ScalarField{ValidData: []bool{false, false}},
+									},
 								},
 								{
 									FieldName: "vec",
 									FieldId:   202,
 									Type:      schemapb.DataType_ArrayOfVector,
-									ValidData: []bool{false, false},
 									Field: &schemapb.FieldData_Vectors{
 										Vectors: &schemapb.VectorField{
-											Dim: 4,
+											ValidData: []bool{false, false},
+											Dim:       4,
 										},
 									},
 								},
@@ -6735,9 +6738,9 @@ func TestCheckAndFlattenStructFieldData_ValidDataMaskMismatch(t *testing.T) {
 								{
 									FieldName: "a",
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{true, false},
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: []bool{true, false},
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6753,9 +6756,9 @@ func TestCheckAndFlattenStructFieldData_ValidDataMaskMismatch(t *testing.T) {
 								{
 									FieldName: "b",
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{false, true},
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: []bool{false, true},
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6824,9 +6827,9 @@ func TestCheckAndFlattenStructFieldData_ValidDataNilVsNonNil(t *testing.T) {
 								{
 									FieldName: "a",
 									Type:      schemapb.DataType_Array,
-									ValidData: nil, // no ValidData
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: nil, // no ValidData
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6842,9 +6845,9 @@ func TestCheckAndFlattenStructFieldData_ValidDataNilVsNonNil(t *testing.T) {
 								{
 									FieldName: "b",
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{true, false}, // has ValidData
 									Field: &schemapb.FieldData_Scalars{
 										Scalars: &schemapb.ScalarField{
+											ValidData: []bool{true, false}, // has ValidData
 											Data: &schemapb.ScalarField_ArrayData{
 												ArrayData: &schemapb.ArrayArray{
 													Data: []*schemapb.ScalarField{
@@ -6914,8 +6917,10 @@ func TestCheckAndFlattenStructFieldData_ValidDataWithoutPayload(t *testing.T) {
 									FieldName: "a",
 									FieldId:   201,
 									Type:      schemapb.DataType_Array,
-									ValidData: []bool{true, false}, // claims row 0 is valid
-									// but no Field/payload
+									Field: &schemapb.FieldData_Scalars{
+										// Claims row 0 is valid, but provides no payload.
+										Scalars: &schemapb.ScalarField{ValidData: []bool{true, false}},
+									},
 								},
 							},
 						},

internal/proxy/validate_util.go

@@ -67,7 +67,7 @@ func validateGeometryFieldSearchResult(fieldData **schemapb.FieldData) error {
 	}
 	wkbArray := (*fieldData).GetScalars().GetGeometryData().GetData()
 	wktArray := make([]string, len(wkbArray))
-	validData := (*fieldData).GetValidData()
+	validData := typeutil.GetFieldDataValidData(*fieldData)
 	for i, data := range wkbArray {
 		if validData != nil && !validData[i] {
 			continue
@@ -94,8 +94,8 @@ func validateGeometryFieldSearchResult(fieldData **schemapb.FieldData) error {
 		},
 		FieldId:   (*fieldData).GetFieldId(),
 		IsDynamic: (*fieldData).GetIsDynamic(),
-		ValidData: (*fieldData).GetValidData(),
 	}
+	typeutil.SetFieldDataValidData(*fieldData, validData)
 	return nil
 }
 
@@ -110,6 +110,9 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, helper *typeutil.Sch
 		return merr.WrapErrServiceInternal("nil schema helper provided for Validation")
 	}
 	for _, field := range data {
+		if !typeutil.ValidateAndNormalizeFieldDataValidData(field) {
+			return merr.WrapErrParameterInvalidMsg("field %s has different legacy and field-specific valid_data", field.GetFieldName())
+		}
 		fieldSchema, err := helper.GetFieldFromName(field.GetFieldName())
 		if err != nil {
 			return err
@@ -212,7 +215,7 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
 		return merr.WrapErrParameterInvalid(schemaDim, dataDim, msg)
 	}
 	getExpectedVectorRows := func(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) uint64 {
-		validData := field.GetValidData()
+		validData := typeutil.GetFieldDataValidData(field)
 		if fieldSchema.GetNullable() && len(validData) > 0 {
 			return uint64(getValidNumber(validData))
 		}
@@ -439,7 +442,7 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
 			if n != numRows {
 				mlog.Warn(context.TODO(), "the num_rows of field is not equal to passed num_rows", mlog.String("fieldName", field.GetFieldName()),
 					mlog.Int64("fieldNumRows", int64(n)), mlog.Int64("passedNumRows", int64(numRows)),
-					mlog.Bools("ValidData", field.GetValidData()))
+					mlog.Bools("ValidData", typeutil.GetFieldDataValidData(field)))
 				return errNumRowsMismatch(field.GetFieldName(), n)
 			}
 		}
@@ -463,8 +466,8 @@ func (v *validateUtil) fillWithValue(data []*schemapb.FieldData, schema *typeuti
 		}
 
 		// adapt all valid data for nullable or default value column
-		if (fieldSchema.GetNullable() || fieldSchema.GetDefaultValue() != nil) && len(field.GetValidData()) == 0 {
-			field.ValidData = lo.RepeatBy(numRows, func(i int) bool { return true })
+		if (fieldSchema.GetNullable() || fieldSchema.GetDefaultValue() != nil) && len(typeutil.GetFieldDataValidData(field)) == 0 {
+			typeutil.SetFieldDataValidData(field, lo.RepeatBy(numRows, func(i int) bool { return true }))
 		}
 
 		if fieldSchema.GetDefaultValue() == nil {
@@ -484,7 +487,11 @@ func (v *validateUtil) fillWithValue(data []*schemapb.FieldData, schema *typeuti
 }
 
 func FillWithNullValue(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema, numRows int) error {
-	err := nullutil.CheckValidData(field.GetValidData(), fieldSchema, numRows)
+	if !typeutil.ValidateAndNormalizeFieldDataValidData(field) {
+		return merr.WrapErrParameterInvalidMsg("field %s has different legacy and field-specific valid_data", field.GetFieldName())
+	}
+	validData := typeutil.GetFieldDataValidData(field)
+	err := nullutil.CheckValidData(validData, fieldSchema, numRows)
 	if err != nil {
 		return err
 	}
@@ -497,65 +504,65 @@ func FillWithNullValue(field *schemapb.FieldData, fieldSchema *schemapb.FieldSch
 	case *schemapb.FieldData_Scalars:
 		switch sd := field.GetScalars().GetData().(type) {
 		case *schemapb.ScalarField_BoolData:
-			sd.BoolData.Data, err = fillWithNullValueImpl(sd.BoolData.Data, field.GetValidData())
+			sd.BoolData.Data, err = fillWithNullValueImpl(sd.BoolData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_IntData:
-			sd.IntData.Data, err = fillWithNullValueImpl(sd.IntData.Data, field.GetValidData())
+			sd.IntData.Data, err = fillWithNullValueImpl(sd.IntData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_LongData:
-			sd.LongData.Data, err = fillWithNullValueImpl(sd.LongData.Data, field.GetValidData())
+			sd.LongData.Data, err = fillWithNullValueImpl(sd.LongData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_FloatData:
-			sd.FloatData.Data, err = fillWithNullValueImpl(sd.FloatData.Data, field.GetValidData())
+			sd.FloatData.Data, err = fillWithNullValueImpl(sd.FloatData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_DoubleData:
-			sd.DoubleData.Data, err = fillWithNullValueImpl(sd.DoubleData.Data, field.GetValidData())
+			sd.DoubleData.Data, err = fillWithNullValueImpl(sd.DoubleData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_TimestamptzData:
-			sd.TimestamptzData.Data, err = fillWithNullValueImpl(sd.TimestamptzData.Data, field.GetValidData())
+			sd.TimestamptzData.Data, err = fillWithNullValueImpl(sd.TimestamptzData.Data, validData)
 			if err != nil {
 				return err
 			}
 		case *schemapb.ScalarField_StringData:
-			sd.StringData.Data, err = fillWithNullValueImpl(sd.StringData.Data, field.GetValidData())
+			sd.StringData.Data, err = fillWithNullValueImpl(sd.StringData.Data, validData)
 			if err != nil {
 				return err
 			}
 		case *schemapb.ScalarField_ArrayData:
-			sd.ArrayData.Data, err = fillWithNullValueImpl(sd.ArrayData.Data, field.GetValidData())
+			sd.ArrayData.Data, err = fillWithNullValueImpl(sd.ArrayData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_JsonData:
-			sd.JsonData.Data, err = fillWithNullValueImpl(sd.JsonData.Data, field.GetValidData())
+			sd.JsonData.Data, err = fillWithNullValueImpl(sd.JsonData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_GeometryData:
-			sd.GeometryData.Data, err = fillWithNullValueImpl(sd.GeometryData.Data, field.GetValidData())
+			sd.GeometryData.Data, err = fillWithNullValueImpl(sd.GeometryData.Data, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_GeometryWktData:
-			sd.GeometryWktData.Data, err = fillWithNullValueImpl(sd.GeometryWktData.Data, field.GetValidData())
+			sd.GeometryWktData.Data, err = fillWithNullValueImpl(sd.GeometryWktData.Data, validData)
 			if err != nil {
 				return err
 			}
@@ -574,7 +581,7 @@ func FillWithNullValue(field *schemapb.FieldData, fieldSchema *schemapb.FieldSch
 			if vectorArray == nil {
 				return merr.WrapErrParameterInvalidMsg("array of vector data is nil, field: %s", field.GetFieldName())
 			}
-			expanded, err := fillVectorArrayNullValueImpl(vectorArray.GetData(), field.GetValidData(), vectorArray.GetDim(), vectorArray.GetElementType())
+			expanded, err := fillVectorArrayNullValueImpl(vectorArray.GetData(), validData, vectorArray.GetDim(), vectorArray.GetElementType())
 			if err != nil {
 				return err
 			}
@@ -613,70 +620,74 @@ func fillVectorArrayNullValueImpl(array []*schemapb.VectorField, validData []boo
 }
 
 func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema, numRows int) error {
+	if !typeutil.ValidateAndNormalizeFieldDataValidData(field) {
+		return merr.WrapErrParameterInvalidMsg("field %s has different legacy and field-specific valid_data", field.GetFieldName())
+	}
 	var err error
+	validData := typeutil.GetFieldDataValidData(field)
 	switch field.Field.(type) {
 	case *schemapb.FieldData_Scalars:
 		switch sd := field.GetScalars().GetData().(type) {
 		case *schemapb.ScalarField_BoolData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetBoolData()
-			sd.BoolData.Data, err = fillWithDefaultValueImpl(sd.BoolData.Data, defaultValue, field.GetValidData())
+			sd.BoolData.Data, err = fillWithDefaultValueImpl(sd.BoolData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_IntData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetIntData()
-			sd.IntData.Data, err = fillWithDefaultValueImpl(sd.IntData.Data, defaultValue, field.GetValidData())
+			sd.IntData.Data, err = fillWithDefaultValueImpl(sd.IntData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_LongData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetLongData()
-			sd.LongData.Data, err = fillWithDefaultValueImpl(sd.LongData.Data, defaultValue, field.GetValidData())
+			sd.LongData.Data, err = fillWithDefaultValueImpl(sd.LongData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_FloatData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetFloatData()
-			sd.FloatData.Data, err = fillWithDefaultValueImpl(sd.FloatData.Data, defaultValue, field.GetValidData())
+			sd.FloatData.Data, err = fillWithDefaultValueImpl(sd.FloatData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_DoubleData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetDoubleData()
-			sd.DoubleData.Data, err = fillWithDefaultValueImpl(sd.DoubleData.Data, defaultValue, field.GetValidData())
+			sd.DoubleData.Data, err = fillWithDefaultValueImpl(sd.DoubleData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_TimestamptzData:
 			// Basic validation: Check if the length of the validity mask matches the number of rows.
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 
 			// Retrieve the default value, which is usually stored as int64 (UTC microseconds).
@@ -699,18 +710,18 @@ func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.Field
 					defaultValue, _ = timestamptz.ValidateAndReturnUnixMicroTz(strDefaultValue, common.DefaultTimezone)
 				}
 			}
-			sd.TimestamptzData.Data, err = fillWithDefaultValueImpl(sd.TimestamptzData.Data, defaultValue, field.GetValidData())
+			sd.TimestamptzData.Data, err = fillWithDefaultValueImpl(sd.TimestamptzData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_StringData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetStringData()
-			sd.StringData.Data, err = fillWithDefaultValueImpl(sd.StringData.Data, defaultValue, field.GetValidData())
+			sd.StringData.Data, err = fillWithDefaultValueImpl(sd.StringData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
@@ -721,20 +732,20 @@ func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.Field
 			return merr.WrapErrParameterInvalid("not set default value", "", "array type not support default value")
 
 		case *schemapb.ScalarField_JsonData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetBytesData()
-			sd.JsonData.Data, err = fillWithDefaultValueImpl(sd.JsonData.Data, defaultValue, field.GetValidData())
+			sd.JsonData.Data, err = fillWithDefaultValueImpl(sd.JsonData.Data, defaultValue, validData)
 			if err != nil {
 				return err
 			}
 
 		case *schemapb.ScalarField_GeometryData:
-			if len(field.GetValidData()) != numRows {
+			if len(validData) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
-				return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
+				return merr.WrapErrParameterInvalid(numRows, len(validData), msg)
 			}
 			defaultValue := fieldSchema.GetDefaultValue().GetStringData()
 			defaultValueWkbBytes, err := common.ConvertWKTToWKB(defaultValue)
@@ -742,7 +753,7 @@ func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.Field
 				mlog.Warn(context.TODO(), "invalid default value for geometry field", mlog.Err(err))
 				return merr.WrapErrParameterInvalidMsg("invalid default value for geometry field")
 			}
-			sd.GeometryData.Data, err = fillWithDefaultValueImpl(sd.GeometryData.Data, defaultValueWkbBytes, field.GetValidData())
+			sd.GeometryData.Data, err = fillWithDefaultValueImpl(sd.GeometryData.Data, defaultValueWkbBytes, validData)
 			if err != nil {
 				return err
 			}
@@ -765,13 +776,13 @@ func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.Field
 			for i := range validData {
 				validData[i] = true
 			}
-			field.ValidData = validData
+			typeutil.SetFieldDataValidData(field, validData)
 		} else {
-			field.ValidData = []bool{}
+			typeutil.SetFieldDataValidData(field, nil)
 		}
 	}
 
-	err = nullutil.CheckValidData(field.GetValidData(), fieldSchema, numRows)
+	err = nullutil.CheckValidData(typeutil.GetFieldDataValidData(field), fieldSchema, numRows)
 	if err != nil {
 		return err
 	}
@@ -946,6 +957,7 @@ func (v *validateUtil) checkTextFieldData(field *schemapb.FieldData, fieldSchema
 
 func (v *validateUtil) checkGeometryFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
 	geometryArray := field.GetScalars().GetGeometryWktData().GetData()
+	validData := typeutil.GetFieldDataValidData(field)
 	wkbArray := make([][]byte, len(geometryArray))
 	if geometryArray == nil && fieldSchema.GetDefaultValue() == nil && !fieldSchema.GetNullable() {
 		msg := fmt.Sprintf("geometry field '%v' is illegal, array type mismatch", field.GetFieldName())
@@ -972,8 +984,8 @@ func (v *validateUtil) checkGeometryFieldData(field *schemapb.FieldData, fieldSc
 		},
 		FieldId:   field.GetFieldId(),
 		IsDynamic: field.GetIsDynamic(),
-		ValidData: field.GetValidData(),
 	}
+	typeutil.SetFieldDataValidData(field, validData)
 	return nil
 }
 

internal/querynodev2/pipeline/insert_node_test.go

@@ -41,6 +41,7 @@ import (
 	"github.com/milvus-io/milvus/pkg/v3/proto/querypb"
 	"github.com/milvus-io/milvus/pkg/v3/proto/segcorepb"
 	"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 type InsertNodeSuite struct {
@@ -298,7 +299,7 @@ func setupSchemaTransitionInsertNodeTest(t *testing.T) (*segments.Manager, *segm
 	require.NoError(t, err)
 	for _, fieldData := range insertMsg.GetFieldsData() {
 		if fieldData.GetFieldId() == schemaTransitionDroppedField {
-			fieldData.ValidData = []bool{true}
+			typeutil.SetFieldDataValidData(fieldData, []bool{true})
 		}
 	}
 	return manager, collection, schemaV950, schemaV951, insertMsg

internal/querynodev2/segments/search_reduce_test.go

@@ -441,11 +441,11 @@ func (suite *SearchReduceSuite) TestResult_SearchGroupByResult() {
 		data := mock_segcore.GenSearchResultData(nq, 2, ids, scores, topks)
 		data.GroupByFieldValues = []*schemapb.FieldData{
 			{
-				FieldId:   101,
-				Type:      schemapb.DataType_Int64,
-				ValidData: []bool{false, false},
+				FieldId: 101,
+				Type:    schemapb.DataType_Int64,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{0, 0}}},
+					ValidData: []bool{false, false},
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{0, 0}}},
 				}},
 			},
 		}

internal/querynodev2/tasks/marshal_reduce_result_test.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/util/function/chain"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 func TestMarshalReduceResult_Basic(t *testing.T) {
@@ -205,7 +206,7 @@ func TestMarshalReduceResult_GroupByNullableValidData(t *testing.T) {
 	require.Len(t, data.GetGroupByFieldValues(), 1)
 	gbv := data.GetGroupByFieldValues()[0]
 	assert.Equal(t, []int64{0, 0}, gbv.GetScalars().GetLongData().GetData())
-	assert.Equal(t, []bool{false, true}, gbv.GetValidData())
+	assert.Equal(t, []bool{false, true}, typeutil.GetFieldDataValidData(gbv))
 	assert.Equal(t, int64(105), gbv.GetFieldId())
 }
 
@@ -232,7 +233,7 @@ func TestMarshalReduceResult_GroupByTimestamptz(t *testing.T) {
 	assert.Equal(t, schemapb.DataType_Timestamptz, gbv.GetType())
 	assert.Equal(t, int64(107), gbv.GetFieldId())
 	assert.Equal(t, []int64{1000, 0}, gbv.GetScalars().GetTimestamptzData().GetData())
-	assert.Equal(t, []bool{true, false}, gbv.GetValidData())
+	assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(gbv))
 	assert.Empty(t, gbv.GetFieldName())
 }
 
@@ -263,7 +264,7 @@ func TestMarshalReduceResult_GroupByGeometry(t *testing.T) {
 		{},
 	}
 	assert.Equal(t, expectedGeometryData, gbv.GetScalars().GetGeometryData().GetData())
-	assert.Equal(t, []bool{true, false}, gbv.GetValidData())
+	assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(gbv))
 	assert.Empty(t, gbv.GetFieldName())
 }
 

internal/querynodev2/tasks/search_task_go_reduce_test.go

@@ -1379,7 +1379,7 @@ func TestExecuteNullableVectorOutput(t *testing.T) {
 	}
 	require.NotNil(t, vecFD, "float_vector must be in output FieldsData")
 	returnedVecs := vecFD.GetVectors().GetFloatVector().GetData()
-	validBits := vecFD.GetValidData()
+	validBits := typeutil.GetFieldDataValidData(vecFD)
 
 	// Ground truth: the inserted FloatVectorFieldData.Data is already compacted
 	// to valid_count * dim (binlog convention for nullable vectors). Map each

internal/storage/insert_data.go

@@ -193,6 +193,7 @@ type FieldData interface {
 	AppendValidDataRows(rows interface{}) error
 	GetDataType() schemapb.DataType
 	GetNullable() bool
+	GetValidData() []bool
 }
 
 func NewFieldData(dataType schemapb.DataType, fieldSchema *schemapb.FieldSchema, cap int) (FieldData, error) {

internal/storage/utils.go

@@ -576,7 +576,7 @@ func validateColumnBasedNullableVectorFieldData(field *schemapb.FieldSchema, src
 		}
 		dim = int64(fieldDim)
 	}
-	requireValidData := logicalRows > 0 || len(srcField.GetValidData()) > 0
+	requireValidData := logicalRows > 0 || len(typeutil.GetFieldDataValidData(srcField)) > 0
 	if err := funcutil.ValidateNullableVectorFieldDataCompactWithDim(srcField, uint64(logicalRows), requireValidData, dim); err != nil {
 		return merr.WrapErrParameterInvalidMsg(err.Error())
 	}
@@ -586,6 +586,12 @@ func validateColumnBasedNullableVectorFieldData(field *schemapb.FieldSchema, src
 func validateColumnBasedInsertMsgNullableVectors(schema *schemapb.CollectionSchema, msg *msgstream.InsertMsg) error {
 	srcFields := make(map[int64]*schemapb.FieldData, len(msg.GetFieldsData()))
 	for _, fieldData := range msg.GetFieldsData() {
+		if !typeutil.ValidateAndNormalizeFieldDataValidData(fieldData) {
+			return merr.WrapErrParameterInvalidMsg(
+				"field %s has different legacy and field-specific valid_data",
+				fieldData.GetFieldName(),
+			)
+		}
 		srcFields[fieldData.GetFieldId()] = fieldData
 	}
 
@@ -608,6 +614,12 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			// unreachable
 			panic("struct is not flattened")
 		}
+		if !typeutil.ValidateAndNormalizeFieldDataValidData(field) {
+			return nil, merr.WrapErrParameterInvalidMsg(
+				"field %s has different legacy and field-specific valid_data",
+				field.GetFieldName(),
+			)
+		}
 		srcFields[field.FieldId] = field
 	}
 
@@ -633,7 +645,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 
 			srcData := srcField.GetVectors().GetFloatVector().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			if err := validateColumnBasedNullableVectorFieldData(field, srcField, int(msg.GetNumRows())); err != nil {
 				return nil, err
 			}
@@ -656,7 +668,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 
 			srcData := srcField.GetVectors().GetBinaryVector()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			if err := validateColumnBasedNullableVectorFieldData(field, srcField, int(msg.GetNumRows())); err != nil {
 				return nil, err
 			}
@@ -679,7 +691,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 
 			srcData := srcField.GetVectors().GetFloat16Vector()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			if err := validateColumnBasedNullableVectorFieldData(field, srcField, int(msg.GetNumRows())); err != nil {
 				return nil, err
 			}
@@ -702,7 +714,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 
 			srcData := srcField.GetVectors().GetBfloat16Vector()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			if err := validateColumnBasedNullableVectorFieldData(field, srcField, int(msg.GetNumRows())); err != nil {
 				return nil, err
 			}
@@ -719,7 +731,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_SparseFloatVector:
 			sparseArray := srcFields[field.FieldID].GetVectors().GetSparseFloatVector()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			var contents [][]byte
 			var dim int64
 			if sparseArray != nil {
@@ -750,7 +762,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 
 			srcData := srcField.GetVectors().GetInt8Vector()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			if err := validateColumnBasedNullableVectorFieldData(field, srcField, int(msg.GetNumRows())); err != nil {
 				return nil, err
 			}
@@ -767,7 +779,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Bool:
 			srcData := srcField.GetScalars().GetBoolData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &BoolFieldData{
 				Data:      srcData,
@@ -777,7 +789,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Int8:
 			srcData := srcField.GetScalars().GetIntData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &Int8FieldData{
 				Data:      lo.Map(srcData, func(v int32, _ int) int8 { return int8(v) }),
@@ -787,7 +799,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Int16:
 			srcData := srcField.GetScalars().GetIntData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &Int16FieldData{
 				Data:      lo.Map(srcData, func(v int32, _ int) int16 { return int16(v) }),
@@ -797,7 +809,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Int32:
 			srcData := srcField.GetScalars().GetIntData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &Int32FieldData{
 				Data:      srcData,
@@ -817,7 +829,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 				}
 			default:
 				srcData := srcField.GetScalars().GetLongData().GetData()
-				validData := srcField.GetValidData()
+				validData := typeutil.GetFieldDataValidData(srcField)
 				fieldData = &Int64FieldData{
 					Data:      srcData,
 					ValidData: validData,
@@ -827,7 +839,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Float:
 			srcData := srcField.GetScalars().GetFloatData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &FloatFieldData{
 				Data:      srcData,
@@ -837,7 +849,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Double:
 			srcData := srcField.GetScalars().GetDoubleData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &DoubleFieldData{
 				Data:      srcData,
@@ -847,7 +859,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Timestamptz:
 			srcData := srcField.GetScalars().GetTimestamptzData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &TimestamptzFieldData{
 				Data:      srcData,
@@ -857,7 +869,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_String, schemapb.DataType_VarChar, schemapb.DataType_Text:
 			srcData := srcField.GetScalars().GetStringData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &StringFieldData{
 				Data:      srcData,
@@ -867,7 +879,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_Array:
 			srcData := srcField.GetScalars().GetArrayData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &ArrayFieldData{
 				ElementType: field.GetElementType(),
@@ -878,7 +890,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_JSON:
 			srcData := srcField.GetScalars().GetJsonData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &JSONFieldData{
 				Data:      srcData,
@@ -888,7 +900,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 
 		case schemapb.DataType_ArrayOfVector:
 			vectorArray := srcField.GetVectors().GetVectorArray()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 
 			fieldData = &VectorArrayFieldData{
 				ElementType: field.GetElementType(),
@@ -899,7 +911,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
 			}
 		case schemapb.DataType_Geometry:
 			srcData := srcField.GetScalars().GetGeometryData().GetData()
-			validData := srcField.GetValidData()
+			validData := typeutil.GetFieldDataValidData(srcField)
 			fieldData = &GeometryFieldData{
 				Data:      srcData,
 				ValidData: validData,
@@ -1403,7 +1415,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Int8FieldData:
 			int32Data := make([]int32, len(rawData.Data))
@@ -1422,7 +1433,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Int16FieldData:
 			int32Data := make([]int32, len(rawData.Data))
@@ -1441,7 +1451,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Int32FieldData:
 			fieldData = &schemapb.FieldData{
@@ -1456,7 +1465,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Int64FieldData:
 			fieldData = &schemapb.FieldData{
@@ -1471,7 +1479,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *FloatFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1486,7 +1493,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *DoubleFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1501,7 +1507,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *TimestamptzFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1516,7 +1521,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *StringFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1531,7 +1535,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *ArrayFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1546,7 +1549,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *JSONFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1561,7 +1563,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *GeometryFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1576,7 +1577,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *FloatVectorFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1592,7 +1592,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: int64(rawData.Dim),
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *BinaryVectorFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1606,7 +1605,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: int64(rawData.Dim),
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Float16VectorFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1620,7 +1618,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: int64(rawData.Dim),
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *BFloat16VectorFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1634,7 +1631,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: int64(rawData.Dim),
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *SparseFloatVectorFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1647,7 +1643,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						},
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *Int8VectorFieldData:
 			dataBytes := arrow.Int8Traits.CastToBytes(rawData.Data)
@@ -1662,7 +1657,6 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: int64(rawData.Dim),
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		case *VectorArrayFieldData:
 			fieldData = &schemapb.FieldData{
@@ -1680,12 +1674,12 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
 						Dim: rawData.Dim,
 					},
 				},
-				ValidData: rawData.ValidData,
 			}
 		default:
 			return insertRecord, merr.WrapErrServiceInternalMsg("unsupported data type when transter storage.InsertData to internalpb.InsertRecord")
 		}
 
+		typeutil.SetFieldDataValidData(fieldData, rawData.GetValidData())
 		insertRecord.FieldsData = append(insertRecord.FieldsData, fieldData)
 		insertRecord.NumRows = int64(rawData.RowNum())
 	}

internal/storage/utils_test.go

@@ -763,7 +763,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -786,7 +786,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -809,7 +809,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -832,7 +832,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -855,7 +855,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -879,7 +879,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -902,7 +902,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -1042,7 +1042,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -1066,7 +1066,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -1089,7 +1089,7 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, dim int
 				FieldId: field.FieldID,
 			}
 			if field.GetNullable() {
-				f.ValidData = testutils.GenerateBoolArray(numRows)
+				typeutil.SetFieldDataValidData(f, testutils.GenerateBoolArray(numRows))
 			}
 			msg.FieldsData = append(msg.FieldsData, f)
 			for _, d := range data {
@@ -1471,6 +1471,69 @@ func TestColumnBasedInsertMsgToInsertDataNullable(t *testing.T) {
 	}
 }
 
+func TestColumnBasedInsertMsgToInsertDataValidDataSources(t *testing.T) {
+	validData := []bool{true, false}
+	schema := &schemapb.CollectionSchema{
+		Fields: []*schemapb.FieldSchema{
+			{
+				FieldID:  100,
+				Name:     "value",
+				DataType: schemapb.DataType_Int64,
+				Nullable: true,
+			},
+		},
+	}
+	makeMsg := func(legacyValidData, scalarValidData []bool) *msgstream.InsertMsg {
+		return &msgstream.InsertMsg{
+			InsertRequest: &msgpb.InsertRequest{
+				NumRows: 2,
+				Version: msgpb.InsertDataVersion_ColumnBased,
+				FieldsData: []*schemapb.FieldData{
+					{
+						FieldId:   100,
+						FieldName: "value",
+						Type:      schemapb.DataType_Int64,
+						ValidData: legacyValidData,
+						Field: &schemapb.FieldData_Scalars{
+							Scalars: &schemapb.ScalarField{
+								ValidData: scalarValidData,
+								Data: &schemapb.ScalarField_LongData{
+									LongData: &schemapb.LongArray{Data: []int64{10, 0}},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+	}
+
+	for _, test := range []struct {
+		name    string
+		legacy  []bool
+		current []bool
+		wantErr bool
+	}{
+		{name: "legacy fallback", legacy: validData},
+		{name: "field-specific", current: validData},
+		{name: "matching dual sources", legacy: validData, current: validData},
+		{name: "mismatched dual sources", legacy: validData, current: []bool{false, true}, wantErr: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			insertData, err := ColumnBasedInsertMsgToInsertData(makeMsg(test.legacy, test.current), schema)
+			if test.wantErr {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), "different legacy and field-specific valid_data")
+				return
+			}
+
+			require.NoError(t, err)
+			fieldData := insertData.Data[100].(*Int64FieldData)
+			assert.Equal(t, validData, fieldData.ValidData)
+		})
+	}
+}
+
 func TestColumnBasedInsertMsgToInsertDataRejectsNullableVectorNonCompactData(t *testing.T) {
 	validData := []bool{true, false, true}
 	makeSchema := func(dataType schemapb.DataType, dim string) *schemapb.CollectionSchema {
@@ -1496,6 +1559,7 @@ func TestColumnBasedInsertMsgToInsertDataRejectsNullableVectorNonCompactData(t *
 		}
 	}
 	makeMsg := func(dataType schemapb.DataType, vectors *schemapb.VectorField) *msgstream.InsertMsg {
+		vectors.ValidData = validData
 		return &msgstream.InsertMsg{
 			InsertRequest: &msgpb.InsertRequest{
 				FieldsData: []*schemapb.FieldData{
@@ -1515,7 +1579,6 @@ func TestColumnBasedInsertMsgToInsertDataRejectsNullableVectorNonCompactData(t *
 						Type:      dataType,
 						FieldName: "vec",
 						FieldId:   101,
-						ValidData: validData,
 						Field:     &schemapb.FieldData_Vectors{Vectors: vectors},
 					},
 				},
@@ -1617,6 +1680,7 @@ func TestColumnBasedInsertMsgToInsertDataRejectsNullableVectorPartialRowData(t *
 		}
 	}
 	makeMsg := func(dataType schemapb.DataType, vectors *schemapb.VectorField) *msgstream.InsertMsg {
+		vectors.ValidData = []bool{true, false}
 		return &msgstream.InsertMsg{
 			InsertRequest: &msgpb.InsertRequest{
 				FieldsData: []*schemapb.FieldData{
@@ -1636,7 +1700,6 @@ func TestColumnBasedInsertMsgToInsertDataRejectsNullableVectorPartialRowData(t *
 						Type:      dataType,
 						FieldName: "vec",
 						FieldId:   101,
-						ValidData: []bool{true, false},
 						Field:     &schemapb.FieldData_Vectors{Vectors: vectors},
 					},
 				},
@@ -3449,7 +3512,8 @@ func TestTransferInsertDataToInsertRecord_NullableArrayOfVector(t *testing.T) {
 	for _, fd := range record.FieldsData {
 		if fd.FieldId == 100 {
 			found = true
-			assert.Equal(t, []bool{true, false}, fd.GetValidData())
+			assert.Nil(t, fd.GetValidData())
+			assert.Equal(t, []bool{true, false}, fd.GetVectors().GetValidData())
 			assert.NotNil(t, fd.GetVectors().GetVectorArray())
 			assert.Equal(t, 2, len(fd.GetVectors().GetVectorArray().GetData()))
 		}

internal/util/function/chain/converter.go

@@ -28,6 +28,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/util/function/chain/types"
 	"github.com/milvus-io/milvus/pkg/v3/util/merr"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 // =============================================================================
@@ -449,7 +450,7 @@ func importFieldDataWithName(builder *DataFrameBuilder, fieldData *schemapb.Fiel
 
 	totalRows := offsets[len(offsets)-1]
 
-	validData := fieldData.GetValidData()
+	validData := typeutil.GetFieldDataValidData(fieldData)
 	nullable := len(validData) > 0
 	if nullable && int64(len(validData)) < totalRows {
 		return merr.WrapErrServiceInternalMsg("field %s: validData length (%d) is less than totalRows (%d)", fieldName, len(validData), totalRows)
@@ -933,7 +934,7 @@ func exportFieldData(df *DataFrame, name string) (*schemapb.FieldData, error) {
 	// Export validity data for nullable fields
 	if df.fieldNullables[name] {
 		if validData := exportValidData(col); validData != nil {
-			fieldData.ValidData = validData
+			typeutil.SetFieldDataValidData(fieldData, validData)
 		}
 	}
 

internal/util/function/chain/converter_test.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/util/function/chain/types"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 // =============================================================================
@@ -920,9 +921,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_Int64() {
 				Type:      schemapb.DataType_Int64,
 				FieldName: "nullable_col",
 				FieldId:   100,
-				ValidData: []bool{true, false, true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true, false, true},
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{10, 0, 30, 0, 50}},
 						},
@@ -971,9 +972,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_String() {
 				Type:      schemapb.DataType_VarChar,
 				FieldName: "nullable_str",
 				FieldId:   100,
-				ValidData: []bool{true, true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, true, false, true},
 						Data: &schemapb.ScalarField_StringData{
 							StringData: &schemapb.StringArray{Data: []string{"a", "b", "", "d"}},
 						},
@@ -1016,9 +1017,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_Int8() {
 				Type:      schemapb.DataType_Int8,
 				FieldName: "nullable_int8",
 				FieldId:   100,
-				ValidData: []bool{false, true, false},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{false, true, false},
 						Data: &schemapb.ScalarField_IntData{
 							IntData: &schemapb.IntArray{Data: []int32{0, 20, 0}},
 						},
@@ -1058,9 +1059,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_Int16() {
 				Type:      schemapb.DataType_Int16,
 				FieldName: "nullable_int16",
 				FieldId:   100,
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true},
 						Data: &schemapb.ScalarField_IntData{
 							IntData: &schemapb.IntArray{Data: []int32{100, 0, 300}},
 						},
@@ -1101,9 +1102,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_MultipleChunks()
 				Type:      schemapb.DataType_Float,
 				FieldName: "nullable_float",
 				FieldId:   100,
-				ValidData: []bool{true, false, true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true, false, true},
 						Data: &schemapb.ScalarField_FloatData{
 							FloatData: &schemapb.FloatArray{Data: []float32{1.1, 0, 3.3, 0, 5.5}},
 						},
@@ -1150,9 +1151,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_AllTypes() {
 				Type:      schemapb.DataType_Bool,
 				FieldName: "nullable_bool",
 				FieldId:   1,
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true},
 						Data: &schemapb.ScalarField_BoolData{
 							BoolData: &schemapb.BoolArray{Data: []bool{true, false, false}},
 						},
@@ -1163,9 +1164,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_AllTypes() {
 				Type:      schemapb.DataType_Int32,
 				FieldName: "nullable_int32",
 				FieldId:   2,
-				ValidData: []bool{false, true, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{false, true, true},
 						Data: &schemapb.ScalarField_IntData{
 							IntData: &schemapb.IntArray{Data: []int32{0, 200, 300}},
 						},
@@ -1176,9 +1177,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableField_AllTypes() {
 				Type:      schemapb.DataType_Double,
 				FieldName: "nullable_double",
 				FieldId:   3,
-				ValidData: []bool{true, true, false},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, true, false},
 						Data: &schemapb.ScalarField_DoubleData{
 							DoubleData: &schemapb.DoubleArray{Data: []float64{1.11, 2.22, 0}},
 						},
@@ -1736,9 +1737,9 @@ func (s *ConverterSuite) TestExportValidData_AllValid() {
 				Type:      schemapb.DataType_Int64,
 				FieldName: "nullable_col",
 				FieldId:   100,
-				ValidData: []bool{true, true, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, true, true},
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
 						},
@@ -1758,7 +1759,7 @@ func (s *ConverterSuite) TestExportValidData_AllValid() {
 
 	// Since all values are valid, exportValidData should return nil → ValidData is nil
 	s.Require().Len(exported.FieldsData, 1)
-	s.Nil(exported.FieldsData[0].ValidData)
+	s.Nil(typeutil.GetFieldDataValidData(exported.FieldsData[0]))
 }
 
 func (s *ConverterSuite) TestFromSearchResultData_NilScalars() {
@@ -1943,9 +1944,9 @@ func (s *ConverterSuite) TestFromSearchResultData_ValidDataTooShort() {
 				Type:      schemapb.DataType_Int64,
 				FieldName: "col",
 				FieldId:   100,
-				ValidData: []bool{true}, // Only 1, need 3
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true}, // Only 1, need 3
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
 						},
@@ -2293,9 +2294,9 @@ func (s *ConverterSuite) TestNullableRoundTrip_WithNulls() {
 				Type:      schemapb.DataType_Int64,
 				FieldName: "nullable_col",
 				FieldId:   100,
-				ValidData: []bool{true, false, true}, // second value is null
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true}, // second value is null
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{10, 0, 30}},
 						},
@@ -2314,8 +2315,9 @@ func (s *ConverterSuite) TestNullableRoundTrip_WithNulls() {
 
 	// ValidData should be exported since there are actual null values
 	s.Require().Len(exported.FieldsData, 1)
-	s.Require().NotNil(exported.FieldsData[0].ValidData)
-	s.Equal([]bool{true, false, true}, exported.FieldsData[0].ValidData)
+	s.Require().NotNil(typeutil.GetFieldDataValidData(exported.FieldsData[0]))
+	s.Equal([]bool{true, false, true}, typeutil.GetFieldDataValidData(exported.FieldsData[0]))
+	s.Nil(exported.FieldsData[0].GetValidData())
 
 	// Verify data values
 	longData := exported.FieldsData[0].GetScalars().GetLongData().GetData()
@@ -2339,9 +2341,9 @@ func (s *ConverterSuite) TestNullableRoundTrip_TimestamptzWithNulls() {
 				Type:      schemapb.DataType_Timestamptz,
 				FieldName: "ts",
 				FieldId:   101,
-				ValidData: []bool{true, false, true},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true},
 						Data: &schemapb.ScalarField_TimestamptzData{
 							TimestamptzData: &schemapb.TimestamptzArray{Data: []int64{1000, 0, 3000}},
 						},
@@ -2362,7 +2364,7 @@ func (s *ConverterSuite) TestNullableRoundTrip_TimestamptzWithNulls() {
 	fieldData := exported.FieldsData[0]
 	s.Equal(schemapb.DataType_Timestamptz, fieldData.GetType())
 	s.Equal([]int64{1000, 0, 3000}, fieldData.GetScalars().GetTimestamptzData().GetData())
-	s.Equal([]bool{true, false, true}, fieldData.GetValidData())
+	s.Equal([]bool{true, false, true}, typeutil.GetFieldDataValidData(fieldData))
 }
 
 func (s *ConverterSuite) TestNullableRoundTrip_BoolWithNulls() {
@@ -2381,9 +2383,9 @@ func (s *ConverterSuite) TestNullableRoundTrip_BoolWithNulls() {
 				Type:      schemapb.DataType_Bool,
 				FieldName: "nullable_bool",
 				FieldId:   100,
-				ValidData: []bool{false, true}, // first is null
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{false, true}, // first is null
 						Data: &schemapb.ScalarField_BoolData{
 							BoolData: &schemapb.BoolArray{Data: []bool{false, true}},
 						},
@@ -2400,8 +2402,8 @@ func (s *ConverterSuite) TestNullableRoundTrip_BoolWithNulls() {
 	exported, err := ToSearchResultData(df)
 	s.Require().NoError(err)
 	s.Require().Len(exported.FieldsData, 1)
-	s.NotNil(exported.FieldsData[0].ValidData)
-	s.Equal([]bool{false, true}, exported.FieldsData[0].ValidData)
+	s.NotNil(typeutil.GetFieldDataValidData(exported.FieldsData[0]))
+	s.Equal([]bool{false, true}, typeutil.GetFieldDataValidData(exported.FieldsData[0]))
 }
 
 func (s *ConverterSuite) TestNullableRoundTrip_StringWithNulls() {
@@ -2420,9 +2422,9 @@ func (s *ConverterSuite) TestNullableRoundTrip_StringWithNulls() {
 				Type:      schemapb.DataType_VarChar,
 				FieldName: "nullable_str",
 				FieldId:   100,
-				ValidData: []bool{true, false}, // second is null
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false}, // second is null
 						Data: &schemapb.ScalarField_StringData{
 							StringData: &schemapb.StringArray{Data: []string{"hello", ""}},
 						},
@@ -2439,8 +2441,8 @@ func (s *ConverterSuite) TestNullableRoundTrip_StringWithNulls() {
 	exported, err := ToSearchResultData(df)
 	s.Require().NoError(err)
 	s.Require().Len(exported.FieldsData, 1)
-	s.NotNil(exported.FieldsData[0].ValidData)
-	s.Equal([]bool{true, false}, exported.FieldsData[0].ValidData)
+	s.NotNil(typeutil.GetFieldDataValidData(exported.FieldsData[0]))
+	s.Equal([]bool{true, false}, typeutil.GetFieldDataValidData(exported.FieldsData[0]))
 }
 
 // =============================================================================
@@ -2521,9 +2523,9 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableInt8() {
 				Type:      schemapb.DataType_Int8,
 				FieldName: "int8_col",
 				FieldId:   100,
-				ValidData: []bool{true, false, true}, // second is null
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false, true}, // second is null
 						Data: &schemapb.ScalarField_IntData{
 							IntData: &schemapb.IntArray{Data: []int32{10, 0, 30}},
 						},
@@ -2549,8 +2551,8 @@ func (s *ConverterSuite) TestFromSearchResultData_NullableInt8() {
 	exported, err := ToSearchResultData(df)
 	s.Require().NoError(err)
 	s.Require().Len(exported.FieldsData, 1)
-	s.NotNil(exported.FieldsData[0].ValidData)
-	s.Equal([]bool{true, false, true}, exported.FieldsData[0].ValidData)
+	s.NotNil(typeutil.GetFieldDataValidData(exported.FieldsData[0]))
+	s.Equal([]bool{true, false, true}, typeutil.GetFieldDataValidData(exported.FieldsData[0]))
 }
 
 // =============================================================================

internal/util/function/chain/dataframe_test.go

@@ -245,9 +245,9 @@ func (s *DataFrameSuite) TestCopyFieldMetadata_IncludesNullable() {
 				Type:      schemapb.DataType_Int64,
 				FieldName: "nullable_col",
 				FieldId:   100,
-				ValidData: []bool{true, false},
 				Field: &schemapb.FieldData_Scalars{
 					Scalars: &schemapb.ScalarField{
+						ValidData: []bool{true, false},
 						Data: &schemapb.ScalarField_LongData{
 							LongData: &schemapb.LongArray{Data: []int64{10, 0}},
 						},

internal/util/function/chain/expr/decay_expr_integration_test.go

@@ -177,9 +177,9 @@ func (s *DecayExprIntegrationTestSuite) TestIntegration_NullDecayFactorTreatedAs
 		Type:      schemapb.DataType_Int64,
 		FieldName: "distance",
 		FieldId:   100,
-		ValidData: []bool{true, false},
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false},
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{
 						Data: []int64{100, 0},

internal/util/queryutil/helpers.go

@@ -240,13 +240,13 @@ func buildMergedFieldData(results []*internalpb.RetrieveResults, selectedRows []
 	if isNullable {
 		validData := make([]bool, len(selectedRows))
 		for i, ref := range selectedRows {
-			vd := results[ref.resultIdx].GetFieldsData()[fieldIdx].GetValidData()
+			vd := typeutil.GetFieldDataValidData(results[ref.resultIdx].GetFieldsData()[fieldIdx])
 			if len(vd) > 0 && int(ref.rowIdx) < len(vd) {
 				validData[i] = vd[ref.rowIdx]
 			}
 			// ValidData absent or rowIdx out of bounds: keep false (null semantics)
 		}
-		newFd.ValidData = validData
+		typeutil.SetFieldDataValidData(newFd, validData)
 	}
 
 	return newFd, nil
@@ -416,7 +416,7 @@ func buildCompactIndices(results []*internalpb.RetrieveResults, fieldIdx int, is
 	for ri, r := range results {
 		numRows := typeutil.GetSizeOfIDs(r.GetIds())
 		fd := r.GetFieldsData()[fieldIdx]
-		vd := fd.GetValidData()
+		vd := typeutil.GetFieldDataValidData(fd)
 
 		if numRows == 0 {
 			indices[ri] = nil
@@ -459,7 +459,7 @@ func arrayOfVectorRowValid(fd *schemapb.FieldData, rowIdx int64, isNullable bool
 			fd.GetFieldId(), fd.GetFieldName(), resultIdx, rowIdx)
 	}
 
-	validData := fd.GetValidData()
+	validData := typeutil.GetFieldDataValidData(fd)
 	if len(validData) == 0 {
 		if isNullable {
 			return false, merr.WrapErrServiceInternalMsg(
@@ -839,7 +839,7 @@ func rangeSliceFieldData(fd *schemapb.FieldData, start, end int) (*schemapb.Fiel
 			Scalars: rangeSliceScalarField(fd.GetScalars(), start, end),
 		}
 	case *schemapb.FieldData_Vectors:
-		vectors, err := rangeSliceVectorField(fd.GetVectors(), start, end, fd.GetValidData())
+		vectors, err := rangeSliceVectorField(fd.GetVectors(), start, end, typeutil.GetFieldDataValidData(fd))
 		if err != nil {
 			return nil, err
 		}
@@ -848,8 +848,8 @@ func rangeSliceFieldData(fd *schemapb.FieldData, start, end int) (*schemapb.Fiel
 		}
 	}
 
-	if len(fd.GetValidData()) > 0 {
-		newFd.ValidData = fd.GetValidData()[start:end]
+	if validData := typeutil.GetFieldDataValidData(fd); len(validData) > 0 {
+		typeutil.SetFieldDataValidData(newFd, validData[start:end])
 	}
 
 	return newFd, nil
@@ -1038,7 +1038,7 @@ func sliceFieldData(fd *schemapb.FieldData, indices []int) (*schemapb.FieldData,
 			Scalars: sliceScalarField(fd.GetScalars(), indices),
 		}
 	case *schemapb.FieldData_Vectors:
-		vectors, err := sliceVectorField(fd.GetVectors(), indices, fd.GetValidData())
+		vectors, err := sliceVectorField(fd.GetVectors(), indices, typeutil.GetFieldDataValidData(fd))
 		if err != nil {
 			return nil, err
 		}
@@ -1048,13 +1048,12 @@ func sliceFieldData(fd *schemapb.FieldData, indices []int) (*schemapb.FieldData,
 	}
 
 	// Preserve ValidData (nullable bitmap) for nullable fields.
-	if len(fd.GetValidData()) > 0 {
-		validData := fd.GetValidData()
+	if validData := typeutil.GetFieldDataValidData(fd); len(validData) > 0 {
 		newValidData := make([]bool, len(indices))
 		for i, idx := range indices {
 			newValidData[i] = validData[idx]
 		}
-		newFd.ValidData = newValidData
+		typeutil.SetFieldDataValidData(newFd, newValidData)
 	}
 
 	return newFd, nil
@@ -1366,7 +1365,7 @@ func newRowSizeCalculator(result *internalpb.RetrieveResults) *rowSizeCalculator
 	}
 	for fieldIdx, fd := range fieldsData {
 		if typeutil.IsCompactNullableVectorFieldData(fd) {
-			indices, _ := typeutil.BuildNullableVectorDataIndices(fd.GetValidData())
+			indices, _ := typeutil.BuildNullableVectorDataIndices(typeutil.GetFieldDataValidData(fd))
 			c.compactIndices[fieldIdx] = indices
 		}
 	}
@@ -1392,7 +1391,7 @@ func calcRowSize(result *internalpb.RetrieveResults, rowIdx int64) int64 {
 func calcFieldElementSize(fd *schemapb.FieldData, rowIdx int) int64 {
 	var compactIdx []int
 	if typeutil.IsCompactNullableVectorFieldData(fd) {
-		compactIdx, _ = typeutil.BuildNullableVectorDataIndices(fd.GetValidData())
+		compactIdx, _ = typeutil.BuildNullableVectorDataIndices(typeutil.GetFieldDataValidData(fd))
 	}
 	return calcFieldElementSizeWithCompactIndex(fd, rowIdx, compactIdx)
 }
@@ -1499,7 +1498,7 @@ func calcFieldElementSizeWithCompactIndex(fd *schemapb.FieldData, rowIdx int, co
 // Returns (value, isNull)
 func getFieldValue(fd *schemapb.FieldData, rowIdx int) (any, bool) {
 	// Check valid_data for nullable fields
-	validData := fd.GetValidData()
+	validData := typeutil.GetFieldDataValidData(fd)
 	if len(validData) > rowIdx && !validData[rowIdx] {
 		return nil, true
 	}
@@ -1569,8 +1568,8 @@ func getRowCount(result *internalpb.RetrieveResults) int {
 	}
 
 	fd := result.GetFieldsData()[0]
-	if len(fd.GetValidData()) > 0 {
-		return len(fd.GetValidData())
+	if validData := typeutil.GetFieldDataValidData(fd); len(validData) > 0 {
+		return len(validData)
 	}
 	if fd.GetScalars() != nil {
 		switch data := fd.GetScalars().GetData().(type) {

internal/util/queryutil/helpers_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 // makeNullableSchema builds a minimal CollectionSchema marking the given
@@ -91,12 +92,12 @@ func TestGetRowCountNullableCompactVectorWithoutIDsUsesLogicalRows(t *testing.T)
 				FieldName: "nullable_vec",
 				FieldId:   100,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{true, false, true},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}},
 					},
 				}},
-				ValidData: []bool{true, false, true},
 			},
 		},
 	}
@@ -116,9 +117,9 @@ func TestBuildMergedFieldData_ValidData(t *testing.T) {
 			{
 				Type: schemapb.DataType_Int64, FieldId: 100,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20}}},
+					ValidData: []bool{true, false}, // row 0 valid, row 1 null
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20}}},
 				}},
-				ValidData: []bool{true, false}, // row 0 valid, row 1 null
 			},
 		},
 	}
@@ -128,9 +129,9 @@ func TestBuildMergedFieldData_ValidData(t *testing.T) {
 			{
 				Type: schemapb.DataType_Int64, FieldId: 100,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{30}}},
+					ValidData: []bool{true},
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{30}}},
 				}},
-				ValidData: []bool{true},
 			},
 		},
 	}
@@ -147,7 +148,7 @@ func TestBuildMergedFieldData_ValidData(t *testing.T) {
 	require.Len(t, merged.FieldsData, 1)
 
 	fd := merged.FieldsData[0]
-	assert.Equal(t, []bool{true, false, true}, fd.ValidData)
+	assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(fd))
 	assert.Equal(t, []int64{10, 20, 30}, fd.GetScalars().GetLongData().GetData())
 }
 
@@ -160,9 +161,9 @@ func TestBuildMergedFieldData_ValidData_MixedNullable(t *testing.T) {
 			{
 				Type: schemapb.DataType_Int64, FieldId: 100,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10}}},
+					ValidData: []bool{false}, // null
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10}}},
 				}},
-				ValidData: []bool{false}, // null
 			},
 		},
 	}
@@ -172,9 +173,9 @@ func TestBuildMergedFieldData_ValidData_MixedNullable(t *testing.T) {
 			{
 				Type: schemapb.DataType_Int64, FieldId: 100,
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{20}}},
+					ValidData: []bool{true}, // explicit valid (segcore always includes ValidData for nullable fields)
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{20}}},
 				}},
-				ValidData: []bool{true}, // explicit valid (segcore always includes ValidData for nullable fields)
 			},
 		},
 	}
@@ -189,7 +190,7 @@ func TestBuildMergedFieldData_ValidData_MixedNullable(t *testing.T) {
 	require.NoError(t, err)
 
 	fd := merged.FieldsData[0]
-	assert.Equal(t, []bool{false, true}, fd.ValidData)
+	assert.Equal(t, []bool{false, true}, typeutil.GetFieldDataValidData(fd))
 }
 
 // =========================================================================
@@ -784,11 +785,11 @@ func TestCalcRowSize_NullableDenseVectorUsesLogicalValidity(t *testing.T) {
 		{
 			name: "float_vector",
 			field: &schemapb.FieldData{
-				Type:      schemapb.DataType_FloatVector,
-				ValidData: []bool{false, true},
+				Type: schemapb.DataType_FloatVector,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2}}},
+					ValidData: []bool{false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2}}},
 				}},
 			},
 			wantSize: 8,
@@ -796,11 +797,11 @@ func TestCalcRowSize_NullableDenseVectorUsesLogicalValidity(t *testing.T) {
 		{
 			name: "binary_vector",
 			field: &schemapb.FieldData{
-				Type:      schemapb.DataType_BinaryVector,
-				ValidData: []bool{false, true},
+				Type: schemapb.DataType_BinaryVector,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  16,
-					Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02}},
+					ValidData: []bool{false, true},
+					Dim:       16,
+					Data:      &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02}},
 				}},
 			},
 			wantSize: 2,
@@ -808,11 +809,11 @@ func TestCalcRowSize_NullableDenseVectorUsesLogicalValidity(t *testing.T) {
 		{
 			name: "float16_vector",
 			field: &schemapb.FieldData{
-				Type:      schemapb.DataType_Float16Vector,
-				ValidData: []bool{false, true},
+				Type: schemapb.DataType_Float16Vector,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04}},
+					ValidData: []bool{false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04}},
 				}},
 			},
 			wantSize: 4,
@@ -820,11 +821,11 @@ func TestCalcRowSize_NullableDenseVectorUsesLogicalValidity(t *testing.T) {
 		{
 			name: "bfloat16_vector",
 			field: &schemapb.FieldData{
-				Type:      schemapb.DataType_BFloat16Vector,
-				ValidData: []bool{false, true},
+				Type: schemapb.DataType_BFloat16Vector,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x01, 0x02, 0x03, 0x04}},
+					ValidData: []bool{false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x01, 0x02, 0x03, 0x04}},
 				}},
 			},
 			wantSize: 4,
@@ -832,11 +833,11 @@ func TestCalcRowSize_NullableDenseVectorUsesLogicalValidity(t *testing.T) {
 		{
 			name: "int8_vector",
 			field: &schemapb.FieldData{
-				Type:      schemapb.DataType_Int8Vector,
-				ValidData: []bool{false, true},
+				Type: schemapb.DataType_Int8Vector,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  4,
-					Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04}},
+					ValidData: []bool{false, true},
+					Dim:       4,
+					Data:      &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04}},
 				}},
 			},
 			wantSize: 4,
@@ -887,10 +888,10 @@ func TestCalcRowSize_NullableSparseVectorUsesCompactMapping(t *testing.T) {
 			r := &internalpb.RetrieveResults{
 				FieldsData: []*schemapb.FieldData{
 					{
-						Type:      schemapb.DataType_SparseFloatVector,
-						ValidData: tt.validData,
+						Type: schemapb.DataType_SparseFloatVector,
 						Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-							Dim: 4,
+							ValidData: tt.validData,
+							Dim:       4,
 							Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 								Dim:      4,
 								Contents: tt.contents,
@@ -1110,9 +1111,9 @@ func TestBuildMergedVectorField_NullableStructVectorArray_AllNull(t *testing.T)
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{false},
+					Dim:       dim,
 				}},
-				ValidData: []bool{false},
 			},
 		},
 	}
@@ -1139,7 +1140,7 @@ func TestBuildMergedVectorField_NullableStructVectorArray_AllNull(t *testing.T)
 
 	merged, err := buildMergedRetrieveResults([]*internalpb.RetrieveResults{result}, []rowRef{{resultIdx: 0, rowIdx: 0}}, schema)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{false}, merged.FieldsData[0].GetValidData())
+	assert.Equal(t, []bool{false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 	va := merged.FieldsData[0].GetVectors().GetVectorArray()
 	require.NotNil(t, va)
 	assert.Equal(t, dim, va.GetDim())
@@ -1158,13 +1159,13 @@ func TestBuildMergedVectorField_NullableStructVectorArray_AllNullWithEmptyVector
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{false, false},
+					Dim:       dim,
 					Data: &schemapb.VectorField_VectorArray{VectorArray: &schemapb.VectorArray{
 						Dim:         dim,
 						ElementType: schemapb.DataType_FloatVector,
 					}},
 				}},
-				ValidData: []bool{false, false},
 			},
 		},
 	}
@@ -1177,7 +1178,7 @@ func TestBuildMergedVectorField_NullableStructVectorArray_AllNullWithEmptyVector
 		schema,
 	)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{false}, merged.FieldsData[0].GetValidData())
+	assert.Equal(t, []bool{false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 	va := merged.FieldsData[0].GetVectors().GetVectorArray()
 	require.NotNil(t, va)
 	assert.Equal(t, dim, va.GetDim())
@@ -1224,14 +1225,14 @@ func TestBuildMergedVectorField_NullableArrayOfVector_RejectsValidDataRowOutOfRa
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{true},
+					Dim:       dim,
 					Data: &schemapb.VectorField_VectorArray{VectorArray: &schemapb.VectorArray{
 						Dim:         dim,
 						Data:        []*schemapb.VectorField{{Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1.0, 2.0}}}}, {Dim: dim, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{3.0, 4.0}}}}},
 						ElementType: schemapb.DataType_FloatVector,
 					}},
 				}},
-				ValidData: []bool{true},
 			},
 		},
 	}
@@ -1256,9 +1257,9 @@ func TestBuildMergedVectorField_NullableArrayOfVectorMixedAllNullSourceKeepsRowD
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{false},
+					Dim:       dim,
 				}},
-				ValidData: []bool{false},
 			},
 		},
 	}
@@ -1269,14 +1270,14 @@ func TestBuildMergedVectorField_NullableArrayOfVectorMixedAllNullSourceKeepsRowD
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{true},
+					Dim:       dim,
 					Data: &schemapb.VectorField_VectorArray{VectorArray: &schemapb.VectorArray{
 						Dim:         dim,
 						Data:        []*schemapb.VectorField{v1},
 						ElementType: schemapb.DataType_FloatVector,
 					}},
 				}},
-				ValidData: []bool{true},
 			},
 		},
 	}
@@ -1289,7 +1290,7 @@ func TestBuildMergedVectorField_NullableArrayOfVectorMixedAllNullSourceKeepsRowD
 		schema,
 	)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{false, true}, merged.FieldsData[0].GetValidData())
+	assert.Equal(t, []bool{false, true}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 	va := merged.FieldsData[0].GetVectors().GetVectorArray()
 	require.Len(t, va.GetData(), 2)
 	assert.Empty(t, va.GetData()[0].GetFloatVector().GetData())
@@ -1310,14 +1311,14 @@ func TestBuildMergedVectorField_NullableArrayOfVectorUsesLogicalRowIndex(t *test
 				Type:    schemapb.DataType_ArrayOfVector,
 				FieldId: fieldID,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: dim,
+					ValidData: []bool{true, false, true, false},
+					Dim:       dim,
 					Data: &schemapb.VectorField_VectorArray{VectorArray: &schemapb.VectorArray{
 						Dim:         dim,
 						Data:        []*schemapb.VectorField{v1, empty, v3, empty},
 						ElementType: schemapb.DataType_FloatVector,
 					}},
 				}},
-				ValidData: []bool{true, false, true, false},
 			},
 		},
 	}
@@ -1330,7 +1331,7 @@ func TestBuildMergedVectorField_NullableArrayOfVectorUsesLogicalRowIndex(t *test
 		schema,
 	)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{true, false, true}, merged.FieldsData[0].GetValidData())
+	assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 
 	va := merged.FieldsData[0].GetVectors().GetVectorArray()
 	require.Len(t, va.GetData(), 3)
@@ -1580,10 +1581,10 @@ func TestBuildCompactIndices_NullableVector(t *testing.T) {
 			{
 				Type: schemapb.DataType_FloatVector, FieldId: 100,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}}},
+					ValidData: []bool{true, false, true}, // row0=valid(idx0), row1=null, row2=valid(idx1)
+					Dim:       2,
+					Data:      &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}}},
 				}},
-				ValidData: []bool{true, false, true}, // row0=valid(idx0), row1=null, row2=valid(idx1)
 			},
 		},
 	}
@@ -1634,7 +1635,9 @@ func TestBuildCompactIndices_FailFast_LengthMismatch(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{
 			{
 				Type: schemapb.DataType_FloatVector, FieldId: 100,
-				ValidData: []bool{true, false}, // 2 entries but 3 rows → mismatch
+				Field: &schemapb.FieldData_Vectors{
+					Vectors: &schemapb.VectorField{ValidData: []bool{true, false}}, // 2 entries but 3 rows -> mismatch
+				},
 			},
 		},
 	}
@@ -1716,11 +1719,11 @@ func TestBuildMergedVectorField_SparseVector_TruncatedContents(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100, Type: schemapb.DataType_SparseFloatVector,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{true, true}, // claims 2 valid rows
 				Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 					Contents: [][]byte{makeTestSparseVec(1, 0.5)}, // only 1 content
 				}},
 			}},
-			ValidData: []bool{true, true}, // claims 2 valid rows
 		}},
 	}
 	selectedRows := []rowRef{{resultIdx: 0, rowIdx: 0}, {resultIdx: 0, rowIdx: 1}}
@@ -1739,10 +1742,10 @@ func TestBuildMergedVectorField_NullableCompact_BinaryVector(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100, Type: schemapb.DataType_BinaryVector,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim:  8,
-				Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0xAA, 0xBB}}, // 2 compact rows
+				ValidData: []bool{true, false, true},
+				Dim:       8,
+				Data:      &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0xAA, 0xBB}}, // 2 compact rows
 			}},
-			ValidData: []bool{true, false, true},
 		}},
 	}
 	selectedRows := []rowRef{{resultIdx: 0, rowIdx: 2}, {resultIdx: 0, rowIdx: 0}, {resultIdx: 0, rowIdx: 1}}
@@ -1753,7 +1756,7 @@ func TestBuildMergedVectorField_NullableCompact_BinaryVector(t *testing.T) {
 
 	// row2 → compact idx 1 → 0xBB, row0 → compact idx 0 → 0xAA, row1 → null (skipped)
 	assert.Equal(t, []byte{0xBB, 0xAA}, merged.FieldsData[0].GetVectors().GetBinaryVector())
-	assert.Equal(t, []bool{true, true, false}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{true, true, false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 }
 
 func TestBuildMergedVectorField_NullableCompact_Float16Vector(t *testing.T) {
@@ -1764,10 +1767,10 @@ func TestBuildMergedVectorField_NullableCompact_Float16Vector(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100, Type: schemapb.DataType_Float16Vector,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim:  1,
-				Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02}}, // 1 compact row
+				ValidData: []bool{false, true},
+				Dim:       1,
+				Data:      &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02}}, // 1 compact row
 			}},
-			ValidData: []bool{false, true},
 		}},
 	}
 	selectedRows := []rowRef{{resultIdx: 0, rowIdx: 1}, {resultIdx: 0, rowIdx: 0}}
@@ -1778,7 +1781,7 @@ func TestBuildMergedVectorField_NullableCompact_Float16Vector(t *testing.T) {
 
 	// row1 → compact idx 0 → [0x01, 0x02], row0 → null (skipped)
 	assert.Equal(t, []byte{0x01, 0x02}, merged.FieldsData[0].GetVectors().GetFloat16Vector())
-	assert.Equal(t, []bool{true, false}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 }
 
 func TestBuildMergedVectorField_Float16Vector_TruncatedData(t *testing.T) {
@@ -1811,10 +1814,10 @@ func TestBuildMergedVectorField_NullableCompact_BFloat16Vector(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100, Type: schemapb.DataType_BFloat16Vector,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim:  1,
-				Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0xAA, 0xBB, 0xCC, 0xDD}},
+				ValidData: []bool{true, false, true},
+				Dim:       1,
+				Data:      &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0xAA, 0xBB, 0xCC, 0xDD}},
 			}},
-			ValidData: []bool{true, false, true},
 		}},
 	}
 	// Select row1(null), row2(valid), row0(valid)
@@ -1826,7 +1829,7 @@ func TestBuildMergedVectorField_NullableCompact_BFloat16Vector(t *testing.T) {
 
 	// row1 → null (skipped), row2 → compact idx 1 → [0xCC,0xDD], row0 → compact idx 0 → [0xAA,0xBB]
 	assert.Equal(t, []byte{0xCC, 0xDD, 0xAA, 0xBB}, merged.FieldsData[0].GetVectors().GetBfloat16Vector())
-	assert.Equal(t, []bool{false, true, true}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{false, true, true}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 }
 
 func TestBuildMergedVectorField_BFloat16Vector_TruncatedData(t *testing.T) {
@@ -1859,10 +1862,10 @@ func TestBuildMergedVectorField_NullableCompact_Int8Vector(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100, Type: schemapb.DataType_Int8Vector,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim:  2,
-				Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x11, 0x22}},
+				ValidData: []bool{false, true},
+				Dim:       2,
+				Data:      &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x11, 0x22}},
 			}},
-			ValidData: []bool{false, true},
 		}},
 	}
 	selectedRows := []rowRef{{resultIdx: 0, rowIdx: 0}, {resultIdx: 0, rowIdx: 1}}
@@ -1873,7 +1876,7 @@ func TestBuildMergedVectorField_NullableCompact_Int8Vector(t *testing.T) {
 
 	// row0 → null (skipped), row1 → compact idx 0 → [0x11,0x22]
 	assert.Equal(t, []byte{0x11, 0x22}, merged.FieldsData[0].GetVectors().GetInt8Vector())
-	assert.Equal(t, []bool{false, true}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{false, true}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 }
 
 func TestBuildMergedVectorField_Int8Vector_TruncatedData(t *testing.T) {
@@ -2238,9 +2241,9 @@ func TestGetFieldValue(t *testing.T) {
 			name: "nullable_null",
 			fd: &schemapb.FieldData{
 				Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-					Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{99}}},
+					ValidData: []bool{false},
+					Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{99}}},
 				}},
-				ValidData: []bool{false},
 			},
 			rowIdx: 0, expected: nil, isNull: true,
 		},
@@ -2743,24 +2746,24 @@ func TestSliceFieldData_ValidData(t *testing.T) {
 	fd := &schemapb.FieldData{
 		Type: schemapb.DataType_Int64, FieldId: 100,
 		Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-			Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}}},
+			ValidData: []bool{true, false, true},
+			Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}}},
 		}},
-		ValidData: []bool{true, false, true},
 	}
 	sliced := requireSliceFieldData(t, fd, []int{2, 0})
-	assert.Equal(t, []bool{true, true}, sliced.GetValidData())
+	assert.Equal(t, []bool{true, true}, typeutil.GetFieldDataValidData(sliced))
 }
 
 func TestRangeSliceFieldData_ValidData(t *testing.T) {
 	fd := &schemapb.FieldData{
 		Type: schemapb.DataType_Int64, FieldId: 100,
 		Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
-			Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}}},
+			ValidData: []bool{true, false, true},
+			Data:      &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}}},
 		}},
-		ValidData: []bool{true, false, true},
 	}
 	sliced := requireRangeSliceFieldData(t, fd, 0, 2)
-	assert.Equal(t, []bool{true, false}, sliced.GetValidData())
+	assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(sliced))
 }
 
 // =========================================================================
@@ -3120,10 +3123,10 @@ func TestBuildMergedVectorField_NullableCompact_FloatVector(t *testing.T) {
 			{
 				Type: schemapb.DataType_FloatVector, FieldId: 100,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}}},
+					ValidData: []bool{true, false, true},
+					Dim:       2,
+					Data:      &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}}},
 				}},
-				ValidData: []bool{true, false, true},
 			},
 		},
 	}
@@ -3133,7 +3136,7 @@ func TestBuildMergedVectorField_NullableCompact_FloatVector(t *testing.T) {
 	require.NoError(t, err)
 	// row2 → compact idx 1 → [3,4], row1 → null (skipped), row0 → compact idx 0 → [1,2]
 	assert.Equal(t, []float32{3, 4, 1, 2}, merged.FieldsData[0].GetVectors().GetFloatVector().GetData())
-	assert.Equal(t, []bool{true, false, true}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{true, false, true}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 }
 
 // =========================================================================
@@ -3243,13 +3246,13 @@ func TestBuildMergedVectorField_NullableSparse_RejectsEmptyValidData(t *testing.
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{true},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{makeTestSparseVec(1, 0.5)},
 					},
 				},
 			}},
-			ValidData: []bool{true},
 		}},
 	}
 
@@ -3261,11 +3264,11 @@ func TestBuildMergedVectorField_NullableSparse_RejectsEmptyValidData(t *testing.
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: nil,
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{Contents: nil},
 				},
 			}},
-			ValidData: nil,
 		}},
 	}
 
@@ -3287,13 +3290,13 @@ func TestBuildMergedVectorField_NullableSparse_RejectsMultipleRowsWithEmptyValid
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{true},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{makeTestSparseVec(1, 0.1)},
 					},
 				},
 			}},
-			ValidData: []bool{true},
 		}},
 	}
 
@@ -3302,11 +3305,11 @@ func TestBuildMergedVectorField_NullableSparse_RejectsMultipleRowsWithEmptyValid
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: nil,
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{Contents: nil},
 				},
 			}},
-			ValidData: nil,
 		}},
 	}
 
@@ -3332,11 +3335,11 @@ func TestBuildMergedVectorField_NullableSparseVector_AllNullWithValidData(t *tes
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{false, false},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{Contents: nil},
 				},
 			}},
-			ValidData: []bool{false, false},
 		}},
 	}
 	results := []*internalpb.RetrieveResults{r}
@@ -3344,7 +3347,7 @@ func TestBuildMergedVectorField_NullableSparseVector_AllNullWithValidData(t *tes
 
 	merged, err := buildMergedRetrieveResults(results, selectedRows, sparseSchema)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{false, false}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{false, false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 	assert.Empty(t, merged.FieldsData[0].GetVectors().GetSparseFloatVector().GetContents())
 }
 
@@ -3358,13 +3361,13 @@ func TestBuildMergedVectorField_NullableSparseVector_Mixed(t *testing.T) {
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{true, false, true},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{makeTestSparseVec(1, 0.1), makeTestSparseVec(3, 0.3)},
 					},
 				},
 			}},
-			ValidData: []bool{true, false, true},
 		}},
 	}
 	results := []*internalpb.RetrieveResults{r}
@@ -3373,7 +3376,7 @@ func TestBuildMergedVectorField_NullableSparseVector_Mixed(t *testing.T) {
 
 	merged, err := buildMergedRetrieveResults(results, selectedRows, sparseSchema)
 	require.NoError(t, err)
-	assert.Equal(t, []bool{true, false}, merged.FieldsData[0].ValidData)
+	assert.Equal(t, []bool{true, false}, typeutil.GetFieldDataValidData(merged.FieldsData[0]))
 	// Only 1 non-null row in output
 	assert.Len(t, merged.FieldsData[0].GetVectors().GetSparseFloatVector().GetContents(), 1)
 }
@@ -3393,13 +3396,13 @@ func TestBuildMergedVectorField_NullableVector_RejectsEmptyValidData(t *testing.
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{true},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{makeTestSparseVec(1, 0.5)},
 					},
 				},
 			}},
-			ValidData: []bool{true},
 		}},
 	}
 	// rB: ValidData absent → contract violation for nullable vector
@@ -3408,11 +3411,11 @@ func TestBuildMergedVectorField_NullableVector_RejectsEmptyValidData(t *testing.
 		FieldsData: []*schemapb.FieldData{{
 			FieldId: 100,
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: nil,
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{Contents: nil},
 				},
 			}},
-			ValidData: nil,
 		}},
 	}
 

internal/util/queryutil/order_op_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/internal/util/reduce/orderby"
 	"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 // makePKField creates a PK field (position 0 in segcore layout).
@@ -113,12 +114,12 @@ func TestOrderByLimitOperator_NullableCompactVectorWithoutIDsUsesLogicalRows(t *
 				FieldName: "nullable_vec",
 				FieldId:   100,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{true, false, true},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}},
 					},
 				}},
-				ValidData: []bool{true, false, true},
 			},
 			{
 				Type:      schemapb.DataType_Int64,
@@ -139,7 +140,7 @@ func TestOrderByLimitOperator_NullableCompactVectorWithoutIDsUsesLogicalRows(t *
 	sorted := outputs[0].(*internalpb.RetrieveResults)
 	require.Len(t, sorted.GetFieldsData(), 2)
 	assert.Equal(t, []int64{10, 20, 30}, sorted.GetFieldsData()[1].GetScalars().GetLongData().GetData())
-	assert.Equal(t, []bool{false, true, true}, sorted.GetFieldsData()[0].GetValidData())
+	assert.Equal(t, []bool{false, true, true}, typeutil.GetFieldDataValidData(sorted.GetFieldsData()[0]))
 	assert.Equal(t, []float32{3, 4, 1, 2}, sorted.GetFieldsData()[0].GetVectors().GetFloatVector().GetData())
 }
 
@@ -562,9 +563,9 @@ func makeNullableInt64Field(fieldID int64, name string, vals []int64, validData
 		Type:      schemapb.DataType_Int64,
 		FieldName: name,
 		FieldId:   fieldID,
-		ValidData: validData,
 		Field: &schemapb.FieldData_Scalars{
 			Scalars: &schemapb.ScalarField{
+				ValidData: validData,
 				Data: &schemapb.ScalarField_LongData{
 					LongData: &schemapb.LongArray{Data: vals},
 				},

internal/util/queryutil/slice_op_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
+	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
 )
 
 func TestSliceOperator_Name(t *testing.T) {
@@ -80,12 +81,12 @@ func TestSliceOperator_NullableCompactVectorWithoutIDsUsesLogicalRows(t *testing
 				FieldName: "nullable_vec",
 				FieldId:   100,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: []bool{true, false, true},
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{
 						FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}},
 					},
 				}},
-				ValidData: []bool{true, false, true},
 			},
 			{
 				Type:      schemapb.DataType_Int64,
@@ -105,7 +106,7 @@ func TestSliceOperator_NullableCompactVectorWithoutIDsUsesLogicalRows(t *testing
 
 	sliced := outputs[0].(*internalpb.RetrieveResults)
 	require.Len(t, sliced.GetFieldsData(), 2)
-	assert.Equal(t, []bool{true}, sliced.GetFieldsData()[0].GetValidData())
+	assert.Equal(t, []bool{true}, typeutil.GetFieldDataValidData(sliced.GetFieldsData()[0]))
 	assert.Equal(t, []float32{3, 4}, sliced.GetFieldsData()[0].GetVectors().GetFloatVector().GetData())
 	assert.Equal(t, []int64{30}, sliced.GetFieldsData()[1].GetScalars().GetLongData().GetData())
 }

pkg/mq/msgstream/msg_test.go

@@ -223,9 +223,9 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_FloatVector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim: 2,
+					ValidData: validData,
+					Dim:       2,
 					Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{
 						Data: []float32{1, 2, 0, 0, 3, 4},
 					}},
@@ -237,10 +237,10 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_BinaryVector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  8,
-					Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{1, 0, 2}},
+					ValidData: validData,
+					Dim:       8,
+					Data:      &schemapb.VectorField_BinaryVector{BinaryVector: []byte{1, 0, 2}},
 				}},
 			},
 		},
@@ -249,10 +249,10 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_Float16Vector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 7, 8}},
+					ValidData: validData,
+					Dim:       2,
+					Data:      &schemapb.VectorField_Float16Vector{Float16Vector: []byte{1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 7, 8}},
 				}},
 			},
 		},
@@ -261,10 +261,10 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_BFloat16Vector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 7, 8}},
+					ValidData: validData,
+					Dim:       2,
+					Data:      &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 7, 8}},
 				}},
 			},
 		},
@@ -273,8 +273,8 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_SparseFloatVector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+					ValidData: validData,
 					Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 						Contents: [][]byte{
 							typeutil.CreateSparseFloatRow([]uint32{1}, []float32{1}),
@@ -290,10 +290,10 @@ func TestInsertMsg_CheckAlignedRejectsNullableVectorNonCompactData(t *testing.T)
 			fieldData: &schemapb.FieldData{
 				FieldName: "vec",
 				Type:      schemapb.DataType_Int8Vector,
-				ValidData: validData,
 				Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-					Dim:  2,
-					Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{1, 2, 0, 0, 3, 4}},
+					ValidData: validData,
+					Dim:       2,
+					Data:      &schemapb.VectorField_Int8Vector{Int8Vector: []byte{1, 2, 0, 0, 3, 4}},
 				}},
 			},
 		},

pkg/util/fastpb/equiv_test.go

@@ -79,6 +79,33 @@ func TestEquiv_FieldData_ValidData(t *testing.T) {
 	})
 }
 
+func TestEquiv_FieldData_FieldSpecificValidData(t *testing.T) {
+	t.Run("scalar", func(t *testing.T) {
+		roundTripFieldData(t, &schemapb.FieldData{
+			Type: schemapb.DataType_Int64,
+			Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.ScalarField_LongData{
+					LongData: &schemapb.LongArray{Data: []int64{1, 0, 3}},
+				},
+			}},
+		})
+	})
+
+	t.Run("vector", func(t *testing.T) {
+		roundTripFieldData(t, &schemapb.FieldData{
+			Type: schemapb.DataType_FloatVector,
+			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				Dim:       2,
+				ValidData: []bool{true, false, true},
+				Data: &schemapb.VectorField_FloatVector{
+					FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}},
+				},
+			}},
+		})
+	})
+}
+
 func TestEquiv_IDs(t *testing.T) {
 	roundTripSRD(t, &schemapb.SearchResultData{Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{10, 20, 30}}}}})
 	roundTripSRD(t, &schemapb.SearchResultData{Ids: &schemapb.IDs{IdField: &schemapb.IDs_StrId{StrId: &schemapb.StringArray{Data: []string{"k1", "k2"}}}}})
@@ -226,7 +253,11 @@ func randFieldData(r *rand.Rand, rows int) *schemapb.FieldData {
 		for i := range vd {
 			vd[i] = r.Intn(2) == 0
 		}
-		fd.ValidData = vd
+		if scalars := fd.GetScalars(); scalars != nil {
+			scalars.ValidData = vd
+		} else {
+			fd.GetVectors().ValidData = vd
+		}
 	}
 	return fd
 }

pkg/util/fastpb/error_paths_test.go

@@ -146,6 +146,10 @@ func TestMalformedWirePerField(t *testing.T) {
 		{"Scalar/mol-smiles-data-malformed", wfield(14, truncTag), newScalarField, decScalarField},
 		{"Scalar/date-data-malformed", wfield(15, truncTag), newScalarField, decScalarField},
 		{"Scalar/time-data-malformed", wfield(16, truncTag), newScalarField, decScalarField},
+		{"Scalar/validdata-truncated-len-prefix", cat(wtag(17, protowire.BytesType), truncTag), newScalarField, decScalarField},
+		{"Scalar/validdata-truncated-packed-varint", wfield(17, truncTag), newScalarField, decScalarField},
+		{"Scalar/validdata-single-varint-ok", protowire.AppendVarint(wtag(17, protowire.VarintType), 1), newScalarField, decScalarField},
+		{"Scalar/validdata-single-varint-truncated", cat(wtag(17, protowire.VarintType), truncTag), newScalarField, decScalarField},
 
 		// --- VectorField: one malformed case per field ---
 		{"Vector/truncated-tag", truncTag, newVectorField, decVectorField},
@@ -162,6 +166,10 @@ func TestMalformedWirePerField(t *testing.T) {
 		{"Vector/int8-truncated-len-prefix", cat(wtag(7, protowire.BytesType), truncTag), newVectorField, decVectorField},
 		{"Vector/vectorarray-len-overruns-buffer", cat(wtag(8, protowire.BytesType), []byte{0x05}), newVectorField, decVectorField},
 		{"Vector/vectorarray-malformed-submsg", wfield(8, truncTag), newVectorField, decVectorField},
+		{"Vector/validdata-truncated-len-prefix", cat(wtag(9, protowire.BytesType), truncTag), newVectorField, decVectorField},
+		{"Vector/validdata-truncated-packed-varint", wfield(9, truncTag), newVectorField, decVectorField},
+		{"Vector/validdata-single-varint-ok", protowire.AppendVarint(wtag(9, protowire.VarintType), 1), newVectorField, decVectorField},
+		{"Vector/validdata-single-varint-truncated", cat(wtag(9, protowire.VarintType), truncTag), newVectorField, decVectorField},
 		{"Vector/unknown-truncated-varint", cat(wtag(20, protowire.VarintType), truncTag), newVectorField, decVectorField},
 
 		// --- IDs ---

pkg/util/fastpb/fielddata.go

@@ -194,9 +194,10 @@ func (d dec) fieldData(b []byte, fd *schemapb.FieldData) error {
 }
 
 // scalarField decodes a wire-format schemapb.ScalarField into sf.
-// Hot array types (bool/int/long/float/double/string/bytes/json) are hand-decoded;
-// rarer variants (array/geometry/timestamptz/mol/date/time/...) decode in-pass
-// via the official codec (decodeScalarFallback) to preserve oneof ordering.
+// Hot array types (bool/int/long/float/double/string/bytes/json) and row
+// validity are hand-decoded; rarer variants
+// (array/geometry/timestamptz/mol/date/time/...) decode in-pass via the
+// official codec (decodeScalarFallback) to preserve oneof ordering.
 func (d dec) scalarField(b []byte, sf *schemapb.ScalarField) error {
 	full := b
 	var rest []byte
@@ -211,6 +212,29 @@ func (d dec) scalarField(b []byte, sf *schemapb.ScalarField) error {
 		if isProto2Group(wtype) {
 			return errProto2
 		}
+		if num == 17 { // ValidData (repeated bool, packed)
+			switch wtype {
+			case 2:
+				v, vn := consumeBytes(b)
+				if vn <= 0 {
+					return errMalformed
+				}
+				if err := appendPackedBool(v, &sf.ValidData); err != nil {
+					return err
+				}
+				b = b[vn:]
+			case 0:
+				v, vn := consumeVarint(b)
+				if vn <= 0 {
+					return errMalformed
+				}
+				sf.ValidData = append(sf.ValidData, v != 0)
+				b = b[vn:]
+			default:
+				return fallbackUnmarshal(full, sf)
+			}
+			continue
+		}
 		if wtype != 2 { // members are all length-delimited; anything else is unknown
 			sn := skipField(b, wtype)
 			if sn <= 0 {
@@ -327,11 +351,15 @@ func unmarshalVectorField(b []byte, vf *schemapb.VectorField) error {
 		if isProto2Group(wtype) {
 			return errProto2
 		}
-		// Known field with an unexpected wire type (Dim=varint; 2..8=length-delimited)
-		// → official codec, to match proto.Unmarshal instead of misdecoding it.
+		// Known field with an unexpected wire type (Dim=varint;
+		// 2..8=length-delimited; ValidData=varint or packed bytes) → official
+		// codec, to match proto.Unmarshal instead of misdecoding it.
 		if (num == 1 && wtype != 0) || (num >= 2 && num <= 8 && wtype != 2) {
 			return fallbackUnmarshal(full, vf)
 		}
+		if num == 9 && wtype != 0 && wtype != 2 {
+			return fallbackUnmarshal(full, vf)
+		}
 		if num >= 2 && num <= 8 {
 			if oneofNum == num && (num == 2 || num == 6 || num == 8) {
 				return fallbackUnmarshal(full, vf)
@@ -407,6 +435,24 @@ func unmarshalVectorField(b []byte, vf *schemapb.VectorField) error {
 			}
 			vf.Data = &schemapb.VectorField_VectorArray{VectorArray: va}
 			b = b[n:]
+		case 9: // ValidData (repeated bool, packed)
+			if wtype == 2 {
+				v, n := consumeBytes(b)
+				if n <= 0 {
+					return errMalformed
+				}
+				if err := appendPackedBool(v, &vf.ValidData); err != nil {
+					return err
+				}
+				b = b[n:]
+			} else {
+				v, n := consumeVarint(b)
+				if n <= 0 {
+					return errMalformed
+				}
+				vf.ValidData = append(vf.ValidData, v != 0)
+				b = b[n:]
+			}
 		default: // unhandled (future) field → fold into official merge
 			n2 := skipField(b, wtype)
 			if n2 <= 0 {

pkg/util/fastpb/message_merge_test.go

@@ -159,6 +159,26 @@ func TestRepeatedMessageOneofMerge(t *testing.T) {
 		assert.True(t, proto.Equal(got, want), "got=%v want=%v", got, want)
 	})
 
+	t.Run("IDs different variants remain last-wins", func(t *testing.T) {
+		for _, msgs := range [][]proto.Message{
+			{
+				&schemapb.IDs{IdField: &schemapb.IDs_UuidId{UuidId: &schemapb.UUIDArray{Data: [][]byte{{1}}}}},
+				&schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{2}}}},
+			},
+			{
+				&schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1}}}},
+				&schemapb.IDs{IdField: &schemapb.IDs_UuidId{UuidId: &schemapb.UUIDArray{Data: [][]byte{{2}}}}},
+			},
+		} {
+			wire := concat(t, msgs...)
+			want := &schemapb.IDs{}
+			require.NoError(t, proto.Unmarshal(wire, want))
+			got := &schemapb.IDs{}
+			require.NoError(t, dec{}.ids(wire, got))
+			assert.True(t, proto.Equal(got, want), "got=%v want=%v", got, want)
+		}
+	})
+
 	t.Run("different variants remain last-wins", func(t *testing.T) {
 		wire := concat(t,
 			&schemapb.FieldData{Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{}}},

pkg/util/fastpb/query_insert.go

@@ -307,9 +307,9 @@ func (d dec) insertRequest(b []byte, ir *milvuspb.InsertRequest) error {
 // UnmarshalUpsertRequest decodes milvuspb.UpsertRequest (write path, client→proxy;
 // UNTRUSTED ingress → strings are UTF-8 validated). Fields 1-8 share InsertRequest's
 // exact wire layout (hot field: fields_data (5)); the upsert-only fields
-// partial_update (9), namespace (10), field_ops (11), and RLS fields 12-13 are
-// folded into the official merge, so they decode via the standard codec and stay
-// wire-equivalent.
+// partial_update (9), namespace (10), field_ops (11), and the RLS fields
+// rls_principal (12) and skip_rls (13) are folded into the official merge, so
+// they decode via the standard codec and stay wire-equivalent.
 func UnmarshalUpsertRequest(b []byte, ur *milvuspb.UpsertRequest) error {
 	proto.Reset(ur) // match official proto.Unmarshal: clear target before decode
 	if err := (dec{utf8: true}).upsertRequest(b, ur); err != nil {
@@ -407,7 +407,7 @@ func (d dec) upsertRequest(b []byte, ur *milvuspb.UpsertRequest) error {
 			if err := appendPackedU32(v, &ur.HashKeys); err != nil {
 				return err
 			}
-		default: // namespace (10), field_ops (11), any future field → official merge
+		default: // fields 10-13 and any future field → official merge
 			rest = append(rest, start[:tn+vn]...)
 		}
 	}

pkg/util/fastpb/query_insert_test.go

@@ -196,11 +196,11 @@ func TestEquiv_UpsertRequest(t *testing.T) {
 		SchemaTimestamp: 123456,
 		PartialUpdate:   true, // field 9 (varint)  → folded to official merge
 		Namespace:       &ns,  // field 10 (bytes)  → folded to official merge
-		RlsPrincipal:    "principal-b",
-		SkipRls:         true,
 		FieldOps: []*schemapb.FieldPartialUpdateOp{ // field 11 (message) → folded
 			{FieldName: "title"},
 		},
+		RlsPrincipal: "principal", // field 12 (bytes)  → folded to official merge
+		SkipRls:      true,        // field 13 (varint) → folded to official merge
 		FieldsData: []*schemapb.FieldData{
 			{Type: schemapb.DataType_VarChar, FieldName: "title", FieldId: 101, Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"x", "y", "z"}}}}}},
 			{Type: schemapb.DataType_FloatVector, FieldName: "emb", FieldId: 102, Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{Dim: 2, Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4, 5, 6}}}}}},

pkg/util/funcutil/func.go

@@ -564,14 +564,15 @@ func ValidateNullableVectorFieldDataCompact(fieldData *schemapb.FieldData, logic
 	if fieldData == nil || !typeutil.IsSupportedNullableVectorType(fieldData.GetType()) {
 		return nil
 	}
-	if len(fieldData.GetValidData()) == 0 && !requireValidData {
+	validData := typeutil.GetFieldDataValidData(fieldData)
+	if len(validData) == 0 && !requireValidData {
 		return nil
 	}
 	physicalRows, err := GetVectorFieldPhysicalRows(fieldData.GetFieldName(), fieldData.GetType(), fieldData.GetVectors())
 	if err != nil {
 		return err
 	}
-	return ValidateNullableVectorCompactRows(fieldData.GetFieldName(), fieldData.GetValidData(), physicalRows, logicalRows, requireValidData)
+	return ValidateNullableVectorCompactRows(fieldData.GetFieldName(), validData, physicalRows, logicalRows, requireValidData)
 }
 
 func ValidateNullableVectorFieldDataCompactWithDim(fieldData *schemapb.FieldData, logicalRows uint64, requireValidData bool, dim int64) error {
@@ -581,14 +582,15 @@ func ValidateNullableVectorFieldDataCompactWithDim(fieldData *schemapb.FieldData
 	if !typeutil.IsSupportedNullableVectorType(fieldData.GetType()) {
 		return nil
 	}
-	if len(fieldData.GetValidData()) == 0 && !requireValidData {
+	validData := typeutil.GetFieldDataValidData(fieldData)
+	if len(validData) == 0 && !requireValidData {
 		return nil
 	}
 	physicalRows, err := getVectorFieldPhysicalRowsWithDim(fieldData.GetFieldName(), fieldData.GetType(), fieldData.GetVectors(), dim)
 	if err != nil {
 		return err
 	}
-	return ValidateNullableVectorCompactRows(fieldData.GetFieldName(), fieldData.GetValidData(), physicalRows, logicalRows, requireValidData)
+	return ValidateNullableVectorCompactRows(fieldData.GetFieldName(), validData, physicalRows, logicalRows, requireValidData)
 }
 
 // GetNumRowOfFieldDataWithSchema returns num of rows with schema specification.
@@ -599,7 +601,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 	if err != nil {
 		return 0, err
 	}
-	if len(fieldData.GetValidData()) > 0 && typeutil.IsSupportedNullableVectorType(fieldSchema.GetDataType()) {
+	validData := typeutil.GetFieldDataValidData(fieldData)
+	if len(validData) > 0 && typeutil.IsSupportedNullableVectorType(fieldSchema.GetDataType()) {
 		dim := fieldData.GetVectors().GetDim()
 		if dim == 0 && fieldSchema.GetDataType() != schemapb.DataType_SparseFloatVector {
 			dim, err = typeutil.GetDim(fieldSchema)
@@ -607,10 +610,10 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 				return 0, err
 			}
 		}
-		if err := ValidateNullableVectorFieldDataCompactWithDim(fieldData, uint64(len(fieldData.GetValidData())), false, dim); err != nil {
+		if err := ValidateNullableVectorFieldDataCompactWithDim(fieldData, uint64(len(validData)), false, dim); err != nil {
 			return 0, err
 		}
-		return uint64(len(fieldData.GetValidData())), nil
+		return uint64(len(validData)), nil
 	}
 	switch fieldSchema.GetDataType() {
 	case schemapb.DataType_Bool:
@@ -640,8 +643,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			fieldNumRows = getNumRowsOfScalarField(fieldData.GetScalars().GetGeometryWktData().GetData())
 		}
 	case schemapb.DataType_FloatVector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			dim := fieldData.GetVectors().GetDim()
 			fieldNumRows, err = GetNumRowsOfFloatVectorField(fieldData.GetVectors().GetFloatVector().GetData(), dim)
@@ -650,8 +653,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			}
 		}
 	case schemapb.DataType_BinaryVector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			dim := fieldData.GetVectors().GetDim()
 			fieldNumRows, err = GetNumRowsOfBinaryVectorField(fieldData.GetVectors().GetBinaryVector(), dim)
@@ -660,8 +663,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			}
 		}
 	case schemapb.DataType_Float16Vector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			dim := fieldData.GetVectors().GetDim()
 			fieldNumRows, err = GetNumRowsOfFloat16VectorField(fieldData.GetVectors().GetFloat16Vector(), dim)
@@ -670,8 +673,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			}
 		}
 	case schemapb.DataType_BFloat16Vector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			dim := fieldData.GetVectors().GetDim()
 			fieldNumRows, err = GetNumRowsOfBFloat16VectorField(fieldData.GetVectors().GetBfloat16Vector(), dim)
@@ -680,14 +683,14 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			}
 		}
 	case schemapb.DataType_SparseFloatVector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			fieldNumRows = uint64(len(fieldData.GetVectors().GetSparseFloatVector().GetContents()))
 		}
 	case schemapb.DataType_Int8Vector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			dim := fieldData.GetVectors().GetDim()
 			fieldNumRows, err = GetNumRowsOfInt8VectorField(fieldData.GetVectors().GetInt8Vector(), dim)
@@ -696,8 +699,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 			}
 		}
 	case schemapb.DataType_ArrayOfVector:
-		if len(fieldData.GetValidData()) > 0 {
-			fieldNumRows = uint64(len(fieldData.GetValidData()))
+		if len(validData) > 0 {
+			fieldNumRows = uint64(len(validData))
 		} else {
 			fieldNumRows = getNumRowsOfArrayVectorField(fieldData.GetVectors().GetVectorArray().GetData())
 		}
@@ -712,6 +715,7 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
 func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
 	var fieldNumRows uint64
 	var err error
+	validData := typeutil.GetFieldDataValidData(fieldData)
 	switch fieldType := fieldData.Field.(type) {
 	case *schemapb.FieldData_Scalars:
 		scalarField := fieldData.GetScalars()
@@ -741,11 +745,11 @@ func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
 		}
 	case *schemapb.FieldData_Vectors:
 		vectorField := fieldData.GetVectors()
-		if len(fieldData.GetValidData()) > 0 {
-			if err := ValidateNullableVectorFieldDataCompact(fieldData, uint64(len(fieldData.GetValidData())), false); err != nil {
+		if len(validData) > 0 {
+			if err := ValidateNullableVectorFieldDataCompact(fieldData, uint64(len(validData)), false); err != nil {
 				return 0, err
 			}
-			return uint64(len(fieldData.GetValidData())), nil
+			return uint64(len(validData)), nil
 		}
 		switch vectorFieldType := vectorField.Data.(type) {
 		case *schemapb.VectorField_FloatVector:

pkg/util/funcutil/func_test.go

@@ -491,9 +491,9 @@ func TestValidateNullableVectorFieldDataCompact(t *testing.T) {
 		fieldData := &schemapb.FieldData{
 			FieldName: "vec",
 			Type:      schemapb.DataType_FloatVector,
-			ValidData: []bool{true, false, true},
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim: 2,
+				ValidData: []bool{true, false, true},
+				Dim:       2,
 				Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{
 					Data: []float32{1, 2, 3, 4},
 				}},
@@ -506,9 +506,9 @@ func TestValidateNullableVectorFieldDataCompact(t *testing.T) {
 		fieldData := &schemapb.FieldData{
 			FieldName: "vec",
 			Type:      schemapb.DataType_FloatVector,
-			ValidData: []bool{true, false, true},
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim: 2,
+				ValidData: []bool{true, false, true},
+				Dim:       2,
 				Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{
 					Data: []float32{1, 2, 3, 4, 5, 6},
 				}},
@@ -523,10 +523,10 @@ func TestValidateNullableVectorFieldDataCompact(t *testing.T) {
 		fieldData := &schemapb.FieldData{
 			FieldName: "vec",
 			Type:      schemapb.DataType_Float16Vector,
-			ValidData: []bool{true, false},
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Dim:  2,
-				Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{1, 2}},
+				ValidData: []bool{true, false},
+				Dim:       2,
+				Data:      &schemapb.VectorField_Float16Vector{Float16Vector: []byte{1, 2}},
 			}},
 		}
 		err := ValidateNullableVectorFieldDataCompact(fieldData, 2, true)
@@ -538,8 +538,8 @@ func TestValidateNullableVectorFieldDataCompact(t *testing.T) {
 		fieldData := &schemapb.FieldData{
 			FieldName: "vec",
 			Type:      schemapb.DataType_SparseFloatVector,
-			ValidData: []bool{false, true, true},
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
+				ValidData: []bool{false, true, true},
 				Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
 					Contents: [][]byte{
 						typeutil.CreateSparseFloatRow([]uint32{1}, []float32{1}),
@@ -572,9 +572,9 @@ func TestValidateNullableVectorFieldDataCompact(t *testing.T) {
 		fieldData := &schemapb.FieldData{
 			FieldName: "vec",
 			Type:      schemapb.DataType_BinaryVector,
-			ValidData: []bool{true, false},
 			Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
-				Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0xff}},
+				ValidData: []bool{true, false},
+				Data:      &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0xff}},
 			}},
 		}
 		require.NoError(t, ValidateNullableVectorFieldDataCompactWithDim(fieldData, 2, true, 8))

pkg/util/funcutil/placeholdergroup_test.go

@@ -66,12 +66,12 @@ func TestFieldDataToPlaceholderGroupBytesWithCount_AllNullSparseVector(t *testin
 		FieldName: "sparse_vec",
 		Field: &schemapb.FieldData_Vectors{
 			Vectors: &schemapb.VectorField{
+				ValidData: []bool{false, false, false},
 				Data: &schemapb.VectorField_SparseFloatVector{
 					SparseFloatVector: &schemapb.SparseFloatArray{},
 				},
 			},
 		},
-		ValidData: []bool{false, false, false},
 	}
 
 	_, valueCount, err := FieldDataToPlaceholderGroupBytesWithCount(fieldData)

pkg/util/typeutil/field_data.go

@@ -1,10 +1,125 @@
 package typeutil
 
 import (
+	"slices"
+
 	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
 	"github.com/milvus-io/milvus/pkg/v3/util/merr"
 )
 
+// FieldData valid_data is handled in three stages:
+//   - At an input boundary, ValidateAndNormalizeFieldDataValidData validates
+//     that the legacy and field-specific values do not conflict, then keeps
+//     only the field-specific representation.
+//   - Internal code uses GetFieldDataValidData and SetFieldDataValidData to
+//     process validity without depending on where it is physically stored.
+//   - At a user-facing response boundary,
+//     ProjectFieldDataValidDataForLegacy restores the legacy representation
+//     while retaining the field-specific value for SDK compatibility.
+
+// GetFieldDataValidData returns the validity of the immediate values carried by
+// FieldData. New payloads store it on ScalarField or VectorField; FieldData is
+// retained as a legacy fallback for older payloads.
+func GetFieldDataValidData(fieldData *schemapb.FieldData) []bool {
+	if legacy := fieldData.GetValidData(); len(legacy) > 0 {
+		return legacy
+	}
+	return getFieldSpecificValidData(fieldData)
+}
+
+// SetFieldDataValidData writes validity to the current field-specific location
+// and clears the legacy FieldData.valid_data source.
+func SetFieldDataValidData(fieldData *schemapb.FieldData, validData []bool) {
+	if fieldData == nil {
+		return
+	}
+
+	if scalars := fieldData.GetScalars(); scalars != nil {
+		scalars.ValidData = validData
+	} else if vectors := fieldData.GetVectors(); vectors != nil {
+		vectors.ValidData = validData
+	} else {
+		return
+	}
+
+	fieldData.ValidData = nil
+}
+
+// ValidateAndNormalizeFieldDataValidData checks the legacy and current
+// validity locations once at an input boundary. Matching values are accepted
+// and normalized to the current field-specific location. It returns false if
+// any immediate or nested FieldData carries different values in both places.
+func ValidateAndNormalizeFieldDataValidData(fieldData *schemapb.FieldData) bool {
+	if !fieldDataValidDataConsistent(fieldData) {
+		return false
+	}
+	normalizeFieldDataValidData(fieldData)
+	return true
+}
+
+func fieldDataValidDataConsistent(fieldData *schemapb.FieldData) bool {
+	if fieldData == nil {
+		return true
+	}
+
+	legacy := fieldData.GetValidData()
+	current := getFieldSpecificValidData(fieldData)
+	if len(legacy) > 0 && len(current) > 0 && !slices.Equal(legacy, current) {
+		return false
+	}
+
+	for _, subField := range fieldData.GetStructArrays().GetFields() {
+		if !fieldDataValidDataConsistent(subField) {
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeFieldDataValidData(fieldData *schemapb.FieldData) {
+	if fieldData == nil {
+		return
+	}
+
+	switch fieldData.Field.(type) {
+	case *schemapb.FieldData_Scalars, *schemapb.FieldData_Vectors:
+		if validData := GetFieldDataValidData(fieldData); len(validData) > 0 {
+			SetFieldDataValidData(fieldData, validData)
+		} else {
+			fieldData.ValidData = nil
+		}
+	case *schemapb.FieldData_StructArrays:
+		fieldData.ValidData = nil
+		for _, subField := range fieldData.GetStructArrays().GetFields() {
+			normalizeFieldDataValidData(subField)
+		}
+	default:
+		fieldData.ValidData = nil
+	}
+}
+
+// ProjectFieldDataValidDataForLegacy copies current top-level validity to the
+// legacy location without clearing the current value. This is intended only
+// for user-facing response boundaries so old SDKs can read the validity mask.
+func ProjectFieldDataValidDataForLegacy(fieldData *schemapb.FieldData) {
+	if fieldData == nil {
+		return
+	}
+	if validData := getFieldSpecificValidData(fieldData); len(validData) > 0 {
+		fieldData.ValidData = slices.Clone(validData)
+	}
+	for _, subField := range fieldData.GetStructArrays().GetFields() {
+		ProjectFieldDataValidDataForLegacy(subField)
+	}
+}
+
+func getFieldSpecificValidData(fieldData *schemapb.FieldData) []bool {
+	if scalars := fieldData.GetScalars(); scalars != nil {
+		return scalars.GetValidData()
+	}
+	return fieldData.GetVectors().GetValidData()
+}
+
 type FieldDataBuilder struct {
 	dt         schemapb.DataType
 	data       []any
@@ -46,9 +161,6 @@ func (b *FieldDataBuilder) Build() *schemapb.FieldData {
 	field := &schemapb.FieldData{
 		Type: b.dt,
 	}
-	if b.hasInvalid {
-		field.ValidData = b.valid
-	}
 
 	switch b.dt {
 	case schemapb.DataType_Bool:
@@ -154,5 +266,8 @@ func (b *FieldDataBuilder) Build() *schemapb.FieldData {
 	default:
 		return nil
 	}
+	if b.hasInvalid {
+		SetFieldDataValidData(field, b.valid)
+	}
 	return field
 }