diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 92e844efca6..8705964ba92 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -470,6 +470,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions scheduler.WithKubeConfig(cc.KubeConfig), scheduler.WithProfiles(cc.ComponentConfig.Profiles...), scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), + scheduler.WithPercentageOfPlacementsToScore(cc.ComponentConfig.PercentageOfPlacementsToScore), scheduler.WithFrameworkOutOfTreeRegistry(outOfTreeRegistry), scheduler.WithPodMaxBackoffSeconds(cc.ComponentConfig.PodMaxBackoffSeconds), scheduler.WithPodInitialBackoffSeconds(cc.ComponentConfig.PodInitialBackoffSeconds), diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 52776fa5e07..1a856930f3e 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -73549,6 +73549,13 @@ func schema_k8sio_kube_scheduler_config_v1_KubeSchedulerConfiguration(ref common Format: "int32", }, }, + "percentageOfPlacementsToScore": { + SchemaProps: spec.SchemaProps{ + Description: "PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and scores them. Valid values are from 0 to 100. The scheduler always tries to find at least one feasible placement. Example: if 500 placements are generated and the value is 30, the scheduler stops after finding 150 feasible placements. When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node counts across all generated placements. If placements overlap, a node present in multiple placements is counted once per placement. The adaptive percentage decreases linearly from 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger sums down to a minimum of 5%. It is overridden by profile level PercentageOfPlacementsToScore. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", + Type: []string{"integer"}, + Format: "int32", + }, + }, "podInitialBackoffSeconds": { SchemaProps: spec.SchemaProps{ Description: "PodInitialBackoffSeconds is the initial backoff for unschedulable pods. If specified, it must be greater than 0. If this value is null, the default value (1s) will be used.", @@ -73639,6 +73646,13 @@ func schema_k8sio_kube_scheduler_config_v1_KubeSchedulerProfile(ref common.Refer Format: "int32", }, }, + "percentageOfPlacementsToScore": { + SchemaProps: spec.SchemaProps{ + Description: "PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and scores them. Valid values are from 0 to 100. The scheduler always tries to find at least one feasible placement. Example: if 500 placements are generated and the value is 30, the scheduler stops after finding 150 feasible placements. When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node counts across all generated placements. If placements overlap, a node present in multiple placements is counted once per placement. The adaptive percentage decreases linearly from 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger sums down to a minimum of 5%. It overrides global PercentageOfPlacementsToScore. If empty, the global PercentageOfPlacementsToScore is used. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", + Type: []string{"integer"}, + Format: "int32", + }, + }, "plugins": { SchemaProps: spec.SchemaProps{ Description: "Plugins specify the set of plugins that should be enabled or disabled. Enabled plugins are the ones that should be enabled in addition to the default plugins. Disabled plugins are any of the default plugins that should be disabled. When no enabled or disabled plugin is specified for an extension point, default plugins for that extension point will be used if there is any. If a QueueSort plugin is specified, the same QueueSort Plugin and PluginConfig must be specified for all profiles.", diff --git a/pkg/scheduler/apis/config/scheme/scheme_test.go b/pkg/scheduler/apis/config/scheme/scheme_test.go index 57711487e1a..e802150d040 100644 --- a/pkg/scheduler/apis/config/scheme/scheme_test.go +++ b/pkg/scheduler/apis/config/scheme/scheme_test.go @@ -219,6 +219,40 @@ profiles: }, }, }, + { + name: "v1 with non-default global percentageOfPlacementsToScore", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +percentageOfPlacementsToScore: 10 +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + PercentageOfPlacementsToScore: nil, + Plugins: defaults.PluginsV1, + PluginConfig: defaults.PluginConfigsV1, + }, + }, + }, + { + name: "v1 with non-default global and profile percentageOfPlacementsToScore", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +percentageOfPlacementsToScore: 10 +profiles: +- percentageOfPlacementsToScore: 20 +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + PercentageOfPlacementsToScore: ptr.To[int32](20), + Plugins: defaults.PluginsV1, + PluginConfig: defaults.PluginConfigsV1, + }, + }, + }, { name: "v1 plugins can include version and kind", data: []byte(` diff --git a/pkg/scheduler/apis/config/types.go b/pkg/scheduler/apis/config/types.go index de3bd832556..5110a50f630 100644 --- a/pkg/scheduler/apis/config/types.go +++ b/pkg/scheduler/apis/config/types.go @@ -69,6 +69,23 @@ type KubeSchedulerConfiguration struct { // nodes will be scored. It is overridden by profile level PercentageOfNodesToScore. PercentageOfNodesToScore *int32 + // PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler + // tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and + // scores them. Valid values are from 0 to 100. + // The scheduler always tries to find at least one feasible placement. + // Example: if 500 placements are generated and the value is 30, the scheduler stops after + // finding 150 feasible placements. + // When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node + // counts across all generated placements. If placements overlap, a node present in multiple + // placements is counted once per placement. The adaptive percentage decreases linearly from + // 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger + // sums down to a minimum of 5%. + // It is overridden by profile level PercentageOfPlacementsToScore. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + PercentageOfPlacementsToScore *int32 + // PodInitialBackoffSeconds is the initial backoff for unschedulable pods. // If specified, it must be greater than 0. If this value is null, the default value (1s) // will be used. @@ -114,6 +131,24 @@ type KubeSchedulerProfile struct { // global PercentageOfNodesToScore will be used. PercentageOfNodesToScore *int32 + // PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler + // tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and + // scores them. Valid values are from 0 to 100. + // The scheduler always tries to find at least one feasible placement. + // Example: if 500 placements are generated and the value is 30, the scheduler stops after + // finding 150 feasible placements. + // When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node + // counts across all generated placements. If placements overlap, a node present in multiple + // placements is counted once per placement. The adaptive percentage decreases linearly from + // 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger + // sums down to a minimum of 5%. + // It overrides global PercentageOfPlacementsToScore. If empty, the global + // PercentageOfPlacementsToScore is used. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + PercentageOfPlacementsToScore *int32 + // Plugins specify the set of plugins that should be enabled or disabled. // Enabled plugins are the ones that should be enabled in addition to the // default plugins. Disabled plugins are any of the default plugins that @@ -229,6 +264,11 @@ const ( // A value of 0 means adaptive, meaning the scheduler figures out a proper default. DefaultPercentageOfNodesToScore = 0 + // DefaultPercentageOfPlacementsToScore is the default percentage of generated placements that + // the scheduler tries to find feasible before it stops searching. + // A value of 0 means adaptive, meaning the scheduler figures out a proper default. + DefaultPercentageOfPlacementsToScore = 0 + // MaxCustomPriorityScore is the max score UtilizationShapePoint expects. MaxCustomPriorityScore int64 = 10 diff --git a/pkg/scheduler/apis/config/v1/defaults.go b/pkg/scheduler/apis/config/v1/defaults.go index ec37cb21118..5e92d9e615f 100644 --- a/pkg/scheduler/apis/config/v1/defaults.go +++ b/pkg/scheduler/apis/config/v1/defaults.go @@ -132,6 +132,10 @@ func SetDefaults_KubeSchedulerConfiguration(obj *configv1.KubeSchedulerConfigura obj.PercentageOfNodesToScore = ptr.To[int32](config.DefaultPercentageOfNodesToScore) } + if obj.PercentageOfPlacementsToScore == nil && feature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) { + obj.PercentageOfPlacementsToScore = ptr.To[int32](config.DefaultPercentageOfPlacementsToScore) + } + if len(obj.LeaderElection.ResourceLock) == 0 { // Use lease-based leader election to reduce cost. // We migrated for EndpointsLease lock in 1.17 and starting in 1.20 we diff --git a/pkg/scheduler/apis/config/v1/defaults_test.go b/pkg/scheduler/apis/config/v1/defaults_test.go index 5cad25790b5..0ddd42f2861 100644 --- a/pkg/scheduler/apis/config/v1/defaults_test.go +++ b/pkg/scheduler/apis/config/v1/defaults_test.go @@ -675,11 +675,51 @@ func TestSchedulerDefaults(t *testing.T) { }, }, }, + { + name: "default TopologyAwareWorkloadScheduling", + features: map[featuregate.Feature]bool{features.GenericWorkload: true, features.TopologyAwareWorkloadScheduling: true}, + config: &configv1.KubeSchedulerConfiguration{}, + expected: &configv1.KubeSchedulerConfiguration{ + Parallelism: ptr.To[int32](16), + DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: &enable, + EnableContentionProfiling: &enable, + }, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: new(true), + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + ResourceLock: "leases", + ResourceNamespace: "kube-system", + ResourceName: "kube-scheduler", + }, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + QPS: 50, + Burst: 100, + ContentType: "application/vnd.kubernetes.protobuf", + }, + PercentageOfNodesToScore: ptr.To[int32](config.DefaultPercentageOfNodesToScore), + PercentageOfPlacementsToScore: ptr.To[int32](config.DefaultPercentageOfPlacementsToScore), + PodInitialBackoffSeconds: ptr.To[int64](1), + PodMaxBackoffSeconds: ptr.To[int64](10), + Profiles: []configv1.KubeSchedulerProfile{ + { + PluginConfig: pluginConfigs, + SchedulerName: new("default-scheduler"), + }, + }, + }, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGatesDuringTest(t, feature.DefaultFeatureGate, tc.features) + if len(tc.features) > 0 && len(tc.expected.Profiles) > 0 { + // Recompute plugin defaults after enabling feature gates for this test. + tc.expected.Profiles[0].Plugins = getDefaultPlugins() + } SetDefaults_KubeSchedulerConfiguration(tc.config) if diff := cmp.Diff(tc.expected, tc.config); diff != "" { t.Errorf("Got unexpected defaults (-want, +got):\n%s", diff) diff --git a/pkg/scheduler/apis/config/v1/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1/zz_generated.conversion.go index 2a4ba24cb64..0b001550e46 100644 --- a/pkg/scheduler/apis/config/v1/zz_generated.conversion.go +++ b/pkg/scheduler/apis/config/v1/zz_generated.conversion.go @@ -419,6 +419,7 @@ func autoConvert_v1_KubeSchedulerConfiguration_To_config_KubeSchedulerConfigurat return err } out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore)) + out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore)) if err := metav1.Convert_Pointer_int64_To_int64(&in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds, s); err != nil { return err } @@ -455,6 +456,7 @@ func autoConvert_config_KubeSchedulerConfiguration_To_v1_KubeSchedulerConfigurat return err } out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore)) + out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore)) if err := metav1.Convert_int64_To_Pointer_int64(&in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds, s); err != nil { return err } @@ -482,6 +484,7 @@ func autoConvert_v1_KubeSchedulerProfile_To_config_KubeSchedulerProfile(in *conf return err } out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore)) + out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore)) if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = new(config.Plugins) @@ -515,6 +518,7 @@ func autoConvert_config_KubeSchedulerProfile_To_v1_KubeSchedulerProfile(in *conf return err } out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore)) + out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore)) if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = new(configv1.Plugins) diff --git a/pkg/scheduler/apis/config/validation/validation.go b/pkg/scheduler/apis/config/validation/validation.go index 4fdc193e213..bd694f31966 100644 --- a/pkg/scheduler/apis/config/validation/validation.go +++ b/pkg/scheduler/apis/config/validation/validation.go @@ -30,6 +30,7 @@ import ( "k8s.io/apiserver/pkg/util/feature" componentbasevalidation "k8s.io/component-base/config/validation" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" schedfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) @@ -68,7 +69,8 @@ func ValidateKubeSchedulerConfiguration(cc *config.KubeSchedulerConfiguration) u errs = append(errs, validateCommonQueueSort(profilesPath, cc.Profiles)...) } - errs = append(errs, validatePercentageOfNodesToScore(field.NewPath("percentageOfNodesToScore"), cc.PercentageOfNodesToScore)) + errs = append(errs, validatePercentage(field.NewPath("percentageOfNodesToScore"), cc.PercentageOfNodesToScore)) + errs = append(errs, validatePercentageOfPlacementsToScore(field.NewPath("percentageOfPlacementsToScore"), cc.PercentageOfPlacementsToScore)) if cc.PodInitialBackoffSeconds <= 0 { errs = append(errs, field.Invalid(field.NewPath("podInitialBackoffSeconds"), @@ -83,15 +85,24 @@ func ValidateKubeSchedulerConfiguration(cc *config.KubeSchedulerConfiguration) u return utilerrors.Flatten(utilerrors.NewAggregate(errs)) } -func validatePercentageOfNodesToScore(path *field.Path, percentageOfNodesToScore *int32) error { - if percentageOfNodesToScore != nil { - if *percentageOfNodesToScore < 0 || *percentageOfNodesToScore > 100 { - return field.Invalid(path, *percentageOfNodesToScore, "not in valid range [0-100]") +func validatePercentage(path *field.Path, percentage *int32) error { + if percentage != nil { + if *percentage < 0 || *percentage > 100 { + return field.Invalid(path, *percentage, "not in valid range [0-100]") } } return nil } +func validatePercentageOfPlacementsToScore(path *field.Path, percentageOfPlacementsToScore *int32) error { + if percentageOfPlacementsToScore != nil { + if !feature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) { + return field.Forbidden(path, "percentageOfPlacementsToScore requires TopologyAwareWorkloadScheduling feature gate to be enabled") + } + } + return validatePercentage(path, percentageOfPlacementsToScore) +} + type invalidPlugins struct { schemeGroupVersion string plugins []string @@ -143,7 +154,8 @@ func validateKubeSchedulerProfile(path *field.Path, apiVersion string, profile * if len(profile.SchedulerName) == 0 { errs = append(errs, field.Required(path.Child("schedulerName"), "")) } - errs = append(errs, validatePercentageOfNodesToScore(path.Child("percentageOfNodesToScore"), profile.PercentageOfNodesToScore)) + errs = append(errs, validatePercentage(path.Child("percentageOfNodesToScore"), profile.PercentageOfNodesToScore)) + errs = append(errs, validatePercentageOfPlacementsToScore(path.Child("percentageOfPlacementsToScore"), profile.PercentageOfPlacementsToScore)) errs = append(errs, validatePluginConfig(path, apiVersion, profile)...) return errs } diff --git a/pkg/scheduler/apis/config/validation/validation_test.go b/pkg/scheduler/apis/config/validation/validation_test.go index 8633fef5765..87ada1a9c6b 100644 --- a/pkg/scheduler/apis/config/validation/validation_test.go +++ b/pkg/scheduler/apis/config/validation/validation_test.go @@ -23,7 +23,10 @@ import ( "github.com/google/go-cmp/cmp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" componentbaseconfig "k8s.io/component-base/config" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" configv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1" "k8s.io/utils/ptr" @@ -109,6 +112,36 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) { percentageOfNodesToScoreNegative := validConfig.DeepCopy() percentageOfNodesToScoreNegative.PercentageOfNodesToScore = ptr.To[int32](-1) + percentageOfPlacementsToScore50 := validConfig.DeepCopy() + percentageOfPlacementsToScore50.PercentageOfPlacementsToScore = ptr.To[int32](50) + + percentageOfPlacementsToScore0 := validConfig.DeepCopy() + percentageOfPlacementsToScore0.PercentageOfPlacementsToScore = ptr.To[int32](0) + + percentageOfPlacementsToScore100 := validConfig.DeepCopy() + percentageOfPlacementsToScore100.PercentageOfPlacementsToScore = ptr.To[int32](100) + + percentageOfPlacementsToScore101 := validConfig.DeepCopy() + percentageOfPlacementsToScore101.PercentageOfPlacementsToScore = ptr.To[int32](101) + + percentageOfPlacementsToScoreNegative := validConfig.DeepCopy() + percentageOfPlacementsToScoreNegative.PercentageOfPlacementsToScore = ptr.To[int32](-1) + + profilePercentageOfPlacementsToScore50 := validConfig.DeepCopy() + profilePercentageOfPlacementsToScore50.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](50) + + profilePercentageOfPlacementsToScore0 := validConfig.DeepCopy() + profilePercentageOfPlacementsToScore0.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](0) + + profilePercentageOfPlacementsToScore100 := validConfig.DeepCopy() + profilePercentageOfPlacementsToScore100.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](100) + + profilePercentageOfPlacementsToScore101 := validConfig.DeepCopy() + profilePercentageOfPlacementsToScore101.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](101) + + profilePercentageOfPlacementsToScoreNegative := validConfig.DeepCopy() + profilePercentageOfPlacementsToScoreNegative.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](-1) + schedulerNameNotSet := validConfig.DeepCopy() schedulerNameNotSet.Profiles[1].SchedulerName = "" @@ -196,8 +229,9 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) { invalidPlugins.Profiles[0].Plugins.Score.Enabled = append(invalidPlugins.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "GCEPDLimits"}) scenarios := map[string]struct { - config *config.KubeSchedulerConfiguration - wantErrs field.ErrorList + featureOverrides featuregatetesting.FeatureOverrides + config *config.KubeSchedulerConfiguration + wantErrs field.ErrorList }{ "good": { config: validConfig, @@ -256,6 +290,110 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) { }, }, }, + "percentage-of-placements-to-score-forbidden-with-tas-disabled": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.TopologyAwareWorkloadScheduling: false, + }, + config: percentageOfPlacementsToScore50, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "percentageOfPlacementsToScore", + }, + }, + }, + "percentage-of-placements-to-score-lower-bound": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: percentageOfPlacementsToScore0, + }, + "percentage-of-placements-to-score-upper-bound": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: percentageOfPlacementsToScore100, + }, + "percentage-of-placements-to-score-greater-than-100": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: percentageOfPlacementsToScore101, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "percentageOfPlacementsToScore", + }, + }, + }, + "percentage-of-placements-to-score-less-than-0": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: percentageOfPlacementsToScoreNegative, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "percentageOfPlacementsToScore", + }, + }, + }, + "profile-percentage-of-placements-to-score-forbidden-with-tas-disabled": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.TopologyAwareWorkloadScheduling: false, + }, + config: profilePercentageOfPlacementsToScore50, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeForbidden, + Field: "profiles[0].percentageOfPlacementsToScore", + }, + }, + }, + "profile-percentage-of-placements-to-score-lower-bound": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: profilePercentageOfPlacementsToScore0, + }, + "profile-percentage-of-placements-to-score-upper-bound": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: profilePercentageOfPlacementsToScore100, + }, + "profile-percentage-of-placements-to-score-greater-than-100": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: profilePercentageOfPlacementsToScore101, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "profiles[0].percentageOfPlacementsToScore", + }, + }, + }, + "profile-percentage-of-placements-to-score-less-than-0": { + featureOverrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }, + config: profilePercentageOfPlacementsToScoreNegative, + wantErrs: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "profiles[0].percentageOfPlacementsToScore", + }, + }, + }, "scheduler-name-not-set": { config: schedulerNameNotSet, wantErrs: field.ErrorList{ @@ -401,6 +539,7 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) { for name, scenario := range scenarios { t.Run(name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, scenario.featureOverrides) errs := ValidateKubeSchedulerConfiguration(scenario.config) diff := cmp.Diff(scenario.wantErrs.ToAggregate(), errs, ignoreBadValueDetail) if diff != "" { diff --git a/pkg/scheduler/apis/config/zz_generated.deepcopy.go b/pkg/scheduler/apis/config/zz_generated.deepcopy.go index 6273c624721..7eaf169dfa9 100644 --- a/pkg/scheduler/apis/config/zz_generated.deepcopy.go +++ b/pkg/scheduler/apis/config/zz_generated.deepcopy.go @@ -198,6 +198,11 @@ func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfigurati *out = new(int32) **out = **in } + if in.PercentageOfPlacementsToScore != nil { + in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore + *out = new(int32) + **out = **in + } if in.Profiles != nil { in, out := &in.Profiles, &out.Profiles *out = make([]KubeSchedulerProfile, len(*in)) @@ -241,6 +246,11 @@ func (in *KubeSchedulerProfile) DeepCopyInto(out *KubeSchedulerProfile) { *out = new(int32) **out = **in } + if in.PercentageOfPlacementsToScore != nil { + in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore + *out = new(int32) + **out = **in + } if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = new(Plugins) diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index 251ac174ba7..d76f09f188e 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -287,6 +287,9 @@ type Framework interface { // HasScorePlugins returns true if at least one Score plugin is defined. HasScorePlugins() bool + // HasPlacementScorePlugins returns true if at least one PlacementScore plugin is defined. + HasPlacementScorePlugins() bool + // PodGroupPostFilterPlugins returns registered PodGroupPostFilter plugins. PodGroupPostFilterPlugins() []fwk.PodGroupPostFilterPlugin @@ -296,6 +299,9 @@ type Framework interface { // PercentageOfNodesToScore returns percentageOfNodesToScore associated to a profile. PercentageOfNodesToScore() *int32 + // PercentageOfPlacementsToScore returns percentageOfPlacementsToScore associated to a profile. + PercentageOfPlacementsToScore() *int32 + // SetPodNominator sets the PodNominator SetPodNominator(nominator fwk.PodNominator) // SetPodActivator sets the PodActivator diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index e2a23f465b9..0d2dac19312 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -95,9 +95,10 @@ type frameworkImpl struct { sharedCSIManager fwk.CSIManager - metricsRecorder *metrics.MetricAsyncRecorder - profileName string - percentageOfNodesToScore *int32 + metricsRecorder *metrics.MetricAsyncRecorder + profileName string + percentageOfNodesToScore *int32 + percentageOfPlacementsToScore *int32 extenders []fwk.Extender fwk.PodNominator @@ -411,6 +412,7 @@ func NewFramework(ctx context.Context, r Registry, profile *config.KubeScheduler f.profileName = profile.SchedulerName f.percentageOfNodesToScore = profile.PercentageOfNodesToScore + f.percentageOfPlacementsToScore = profile.PercentageOfPlacementsToScore if profile.Plugins == nil { return f, nil } @@ -427,10 +429,11 @@ func NewFramework(ctx context.Context, r Registry, profile *config.KubeScheduler pluginConfig[name] = profile.PluginConfig[i].Args } outputProfile := config.KubeSchedulerProfile{ - SchedulerName: f.profileName, - PercentageOfNodesToScore: f.percentageOfNodesToScore, - Plugins: profile.Plugins, - PluginConfig: make([]config.PluginConfig, 0, len(pg)), + SchedulerName: f.profileName, + PercentageOfNodesToScore: f.percentageOfNodesToScore, + PercentageOfPlacementsToScore: f.percentageOfPlacementsToScore, + Plugins: profile.Plugins, + PluginConfig: make([]config.PluginConfig, 0, len(pg)), } f.pluginsMap = make(map[string]fwk.Plugin) @@ -2370,6 +2373,11 @@ func (f *frameworkImpl) HasScorePlugins() bool { return len(f.scorePlugins) > 0 } +// HasPlacementScorePlugins returns true if at least one placementScore plugin is defined. +func (f *frameworkImpl) HasPlacementScorePlugins() bool { + return len(f.placementScorePlugins) > 0 +} + // PodGroupPostFilterPlugins returns registered PodGroupPostFilter plugins. func (f *frameworkImpl) PodGroupPostFilterPlugins() []fwk.PodGroupPostFilterPlugin { return f.podGroupPostFilterPlugins @@ -2470,6 +2478,11 @@ func (f *frameworkImpl) PercentageOfNodesToScore() *int32 { return f.percentageOfNodesToScore } +// PercentageOfPlacementsToScore returns percentageOfPlacementsToScore associated to a profile. +func (f *frameworkImpl) PercentageOfPlacementsToScore() *int32 { + return f.percentageOfPlacementsToScore +} + // Parallelizer returns a parallelizer holding parallelism for scheduler. func (f *frameworkImpl) Parallelizer() fwk.Parallelizer { return f.parallelizer diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index 27e4fb73441..d2ff951400d 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -72,9 +72,10 @@ const ( placementGeneratePlugin = "placement-generate-plugin" defaultPreemptionPlugin = names.DefaultPreemption - testProfileName = "test-profile" - testPercentageOfNodesToScore = 35 - nodeName = "testNode" + testProfileName = "test-profile" + testPercentageOfNodesToScore = 35 + testPercentageOfPlacementsToScore = 10 + nodeName = "testNode" injectReason = "injected status" injectFilterReason = "injected filter status" @@ -4636,9 +4637,10 @@ func TestRecordingMetrics(t *testing.T) { recorder := metrics.NewMetricsAsyncRecorder(100, time.Nanosecond, ctx.Done()) profile := config.KubeSchedulerProfile{ - PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore), - SchedulerName: testProfileName, - Plugins: plugins, + PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore), + PercentageOfPlacementsToScore: ptr.To[int32](testPercentageOfPlacementsToScore), + SchedulerName: testProfileName, + Plugins: plugins, } f, err := newFrameworkWithQueueSortAndBind(ctx, r, profile, withMetricsRecorder(recorder), @@ -4759,9 +4761,10 @@ func TestRunBindPlugins(t *testing.T) { ctx, cancel := context.WithCancel(ctx) recorder := metrics.NewMetricsAsyncRecorder(100, time.Nanosecond, ctx.Done()) profile := config.KubeSchedulerProfile{ - SchedulerName: testProfileName, - PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore), - Plugins: plugins, + SchedulerName: testProfileName, + PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore), + PercentageOfPlacementsToScore: ptr.To[int32](testPercentageOfPlacementsToScore), + Plugins: plugins, } fwk, err := newFrameworkWithQueueSortAndBind(ctx, r, profile, withMetricsRecorder(recorder)) if err != nil { diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 58bbe4d4c73..105f650a821 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -22,6 +22,7 @@ import ( "iter" "maps" "math/rand" + "slices" "time" v1 "k8s.io/api/core/v1" @@ -42,6 +43,16 @@ import ( "k8s.io/utils/ptr" ) +const ( + // minFeasiblePlacementsToFind is the minimum number of placements that would be scored + // in each scheduling cycle. + minFeasiblePlacementsToFind = 1 + // minFeasiblePlacementsPercentageToFind is the minimum adaptive percentage used when + // PercentageOfPlacementsToScore is 0. An explicitly configured lower value, such as 3, + // overrides this adaptive minimum. + minFeasiblePlacementsPercentageToFind = 5 +) + // scheduleOnePodGroup does the entire workload-aware scheduling workflow for a single pod group. func (sched *Scheduler) scheduleOnePodGroup(ctx context.Context, podGroupInfo *framework.QueuedPodGroupInfo) { logger := klog.FromContext(ctx) @@ -927,9 +938,8 @@ func (sched *Scheduler) updateCompositePodGroupCondition(ctx context.Context, // First it runs placement generator plugins to create a list of placements. // Placement is a set of nodes that will be considered when scheduling a pod group. // For a standalone PodGroup it evaluates the placement matching the pods' NominatedNodeName first -// and uses it if the gang is feasible there, short-circuiting the rest. Otherwise (or for a -// PodGroup that is part of a CompositePodGroup) it tries every placement through -// podGroupSchedulingDefaultAlgorithm and runs placement scorer plugins to select the best one. +// and uses it if the gang is feasible there, short-circuiting the rest. Otherwise it evaluates +// candidates up to the feasible-placement limit and runs placement scorer plugins to select the best one. func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, podGroupInfo *framework.PodGroupInfo, queuedPodGroupInfo *framework.QueuedPodGroupInfo) (finalResult *podGroupAlgorithmResult, revertFns revertFns) { allNodes, err := sched.nodeInfoSnapshot.ListNodesInPlacement() if err != nil { @@ -972,7 +982,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context // CompositePodGroup defers its feasibility verdict to the CPG root, where a Success status // with nothing scheduled is still meaningful. Short-circuiting on it (or dropping it) here // could wrongly report the whole CPG Unschedulable and stop sibling groups from being - // evaluated, so CPG children evaluate every placement as before. + // evaluated, so CPG children use the regular placement search. // TODO(kubernetes/kubernetes#140863): extend NNN support to CompositePodGroups. nominatedFeasible := false var nominated *fwk.Placement @@ -1001,6 +1011,8 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context // Only evaluate the remaining placements when the nominated one wasn't feasible. if !nominatedFeasible { + var numPlacementsToFind int + placements, numPlacementsToFind = sched.placementsToEvaluate(schedFwk, placements) for _, placement := range placements { if placement == nominated { continue @@ -1017,6 +1029,10 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context if result.status.IsSuccess() { successfulResults[placement] = result } + + if len(successfulResults) >= numPlacementsToFind { + break + } } } @@ -1061,7 +1077,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context // compositePodGroupSchedulingPlacementAlgorithm tries several different combinations for scheduling the child pod groups and selects the best one. // First it runs placement generator plugins to create a list of placements. // Placement is a set of nodes that will be considered when scheduling a pod group. -// Then for each placement it tries to schedule the pod group through podGroupSchedulingDefaultAlgorithm. +// Then it evaluates candidate placements through compositePodGroupSchedulingDefaultAlgorithm up to the feasible-placement limit. // Finally, it runs placement scorer plugins to select the best placement. func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, root *framework.QueuedPodGroupInfo, podGroupInfo *framework.PodGroupInfo, results map[fwk.EntityKey]*podGroupAlgorithmResult) (finalResult *podGroupAlgorithmResult, revertFns revertFns) { defer func() { @@ -1089,6 +1105,8 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex var anyResultSubtree map[fwk.EntityKey]*podGroupAlgorithmResult successfulResults := make(map[*fwk.Placement]map[fwk.EntityKey]*podGroupAlgorithmResult) + placements, numPlacementsToFind := sched.placementsToEvaluate(schedFwk, placements) + parentPlacement := sched.nodeInfoSnapshot.GetPlacement() defer func() { sched.nodeInfoSnapshot.ForgetPlacement() @@ -1129,6 +1147,10 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex if result.status.IsSuccess() { successfulResults[placement] = subtreeResult } + + if len(successfulResults) >= numPlacementsToFind { + break + } } if len(successfulResults) == 0 { @@ -1168,6 +1190,63 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex return bestResult[podGroupInfo.GetKey()], revertFns } +// randShufflePlacements is the default placement shuffler, randomizing placement order in place. +func randShufflePlacements(placements []*fwk.Placement) { + rand.Shuffle(len(placements), func(i, j int) { + placements[i], placements[j] = placements[j], placements[i] + }) +} + +// placementsToEvaluate returns the placements in evaluation order and the number of feasible +// placements to find before the search stops. +func (sched *Scheduler) placementsToEvaluate(schedFwk framework.Framework, placements []*fwk.Placement) ([]*fwk.Placement, int) { + numPlacementsToFind := 1 + if schedFwk.HasPlacementScorePlugins() { + numPlacementsToFind = sched.numFeasiblePlacementsToFind(schedFwk.PercentageOfPlacementsToScore(), placements) + } + + // Only a subset of placements will be evaluated, so shuffle to avoid favoring the order + // returned by the PlacementGenerate plugin. Clone the slice to avoid mutating plugin data. + if sched.shufflePlacements != nil && numPlacementsToFind < len(placements) { + placements = slices.Clone(placements) + sched.shufflePlacements(placements) + } + return placements, numPlacementsToFind +} + +// numFeasiblePlacementsToFind returns the number of feasible placements that once found, the scheduler stops +// its search for more feasible placements. +func (sched *Scheduler) numFeasiblePlacementsToFind(percentageOfPlacementsToScore *int32, placements []*fwk.Placement) (numPlacements int) { + numAllPlacements := len(placements) + + if numAllPlacements < minFeasiblePlacementsToFind { + return numAllPlacements + } + + // Use profile percentageOfPlacementsToScore if it's set. Otherwise, use global percentageOfPlacementsToScore. + var percentage int + if percentageOfPlacementsToScore != nil { + percentage = int(*percentageOfPlacementsToScore) + } else { + percentage = int(sched.percentageOfPlacementsToScore) + } + + // If percentage is 0, linearly interpolate from 100% to 10% as the summed node count + // across generated placements grows from 0 to 5000, with a 5% floor. + if percentage == 0 { + numAllNodes := 0 + for _, placement := range placements { + // Placements may overlap or omit cluster nodes, so use their summed node count. + numAllNodes += len(placement.Nodes) + } + percentage = max(minFeasiblePlacementsPercentageToFind, 100-numAllNodes*9/500) + } + + numPlacements = max(minFeasiblePlacementsToFind, numAllPlacements*percentage/100) + + return numPlacements +} + func (sched *Scheduler) findBestPodGroupPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.PodGroupInfo, successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) (*fwk.Placement, *fwk.Status) { if len(successfulResults) == 1 { for placement := range successfulResults { @@ -1227,8 +1306,7 @@ func (sched *Scheduler) evaluatePlacement(ctx context.Context, schedFwk framewor // // The framework contract permits overlapping placements, so a nominated node can appear in more // than one. NNN alone can't tell which placement WAP picked last cycle, so we only take the fast -// path when exactly one placement matches; otherwise the caller scores every placement, which is -// deterministic regardless of the order the generator returned them. +// path when exactly one placement matches; otherwise the caller uses the regular placement search. func nominatedPlacement(placements []*fwk.Placement, podGroupInfo *framework.PodGroupInfo, queuedPodGroupInfo *framework.QueuedPodGroupInfo) *fwk.Placement { // Collect nominated nodes across the pods of the currently evaluated pod group node // (podGroupInfo), which for CPG TAS is the CPG or PG carrying the TAS constraints, not the @@ -1248,7 +1326,7 @@ func nominatedPlacement(placements []*fwk.Placement, podGroupInfo *framework.Pod for _, node := range placement.Nodes { if nominatedNodes.Has(node.Node().Name) { if matched != nil { - // Overlap: fall back to scoring all placements. + // Overlap: fall back to the regular placement search. return nil } matched = placement diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index f79fde94217..798404ece49 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -68,6 +68,7 @@ import ( st "k8s.io/kubernetes/pkg/scheduler/testing" tf "k8s.io/kubernetes/pkg/scheduler/testing/framework" testingclock "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" ) // fakePodGroupPlugin simulates Filter, PostFilter, Permit and PodGroupPostFilter behaviors for PodGroup scheduling testing. @@ -3575,8 +3576,7 @@ func TestPodGroupSchedulingPlacementAlgorithm_NominatedNode(t *testing.T) { SchedulingQueue: queue, Profiles: profile.Map{"test-scheduler": schedFwk}, } - sched.initAlgorithm() - sched.SchedulePod = sched.algorithm.SchedulePod + initTestAlgorithm(t, sched) if err := sched.Cache.UpdateSnapshot(logger, sched.nodeInfoSnapshot); err != nil { t.Fatalf("Failed to update snapshot: %v", err) @@ -8679,3 +8679,611 @@ func TestSubmitCompositePodGroupAlgorithmResult_StatusUpdates(t *testing.T) { }) } } + +func TestNumFeasiblePlacementsToFind(t *testing.T) { + makePlacements := func(placementsCount, nodesCount int) []*fwk.Placement { + res := make([]*fwk.Placement, placementsCount) + j := 0 + for i := range placementsCount { + nodes := make([]fwk.NodeInfo, 0) + for ; j < nodesCount*(i+1)/placementsCount; j++ { + nodes = append(nodes, framework.NewNodeInfo()) + } + res[i] = &fwk.Placement{ + Nodes: nodes, + } + } + return res + } + + tests := []struct { + name string + globalPercentage int32 + profilePercentage *int32 + placementsCount int + nodesCount int + wantNumPlacements int + }{ + { + name: "default limit for small cluster and small number of placements", + placementsCount: 10, + nodesCount: 100, + wantNumPlacements: 9, + }, + { + name: "default limit for large cluster and small number of placements", + placementsCount: 20, + nodesCount: 5000, + wantNumPlacements: 2, + }, + { + name: "default limit for small cluster and large number of placements", + placementsCount: 100, + nodesCount: 100, + wantNumPlacements: 99, + }, + { + name: "default limit for large cluster and large number of placements", + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 100, + }, + { + name: "default limit does not drop below 5%", + placementsCount: 10000, + nodesCount: 50000, + wantNumPlacements: 500, + }, + { + name: "non-default limit can drop below 5%", + globalPercentage: 1, + placementsCount: 10000, + nodesCount: 50000, + wantNumPlacements: 100, + }, + { + name: "limit cannot drop below 1 placement", + globalPercentage: 1, + placementsCount: 10, + nodesCount: 50000, + wantNumPlacements: 1, + }, + { + name: "both global and profile limit set, profile limit lower than global", + globalPercentage: 50, + profilePercentage: ptr.To[int32](5), + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 50, + }, + { + name: "both global and profile limit set, profile limit higher than global", + globalPercentage: 50, + profilePercentage: ptr.To[int32](55), + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 550, + }, + { + name: "both global and profile limit set, profile limit using default", + globalPercentage: 50, + profilePercentage: ptr.To[int32](0), // use default limit + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 100, + }, + { + name: "only profile limit set", + profilePercentage: ptr.To[int32](5), + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 50, + }, + { + name: "only global limit set", + globalPercentage: 50, + placementsCount: 1000, + nodesCount: 5000, + wantNumPlacements: 500, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sched := &Scheduler{ + percentageOfPlacementsToScore: tt.globalPercentage, + } + placements := makePlacements(tt.placementsCount, tt.nodesCount) + if got := sched.numFeasiblePlacementsToFind(tt.profilePercentage, placements); got != tt.wantNumPlacements { + t.Errorf("Scheduler.numFeasiblePlacementsToFind() = %v, want %v", got, tt.wantNumPlacements) + } + }) + } +} + +type trackingPlacementScorePlugin struct { + lock sync.Mutex + scoredPlacements sets.Set[string] +} + +var _ fwk.PlacementScorePlugin = &trackingPlacementScorePlugin{} + +func (t *trackingPlacementScorePlugin) Name() string { + return "trackingPlacementScorePlugin" +} + +func (t *trackingPlacementScorePlugin) PlacementScoreExtensions() fwk.PlacementScoreExtensions { + return nil +} + +func (t *trackingPlacementScorePlugin) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { + t.lock.Lock() + defer t.lock.Unlock() + t.scoredPlacements.Insert(placement.Name) + return 1, nil +} + +type trackingFilterPlugin struct { + lock sync.Mutex + scoredNodes sets.Set[string] +} + +var _ fwk.FilterPlugin = &trackingFilterPlugin{} + +func (t *trackingFilterPlugin) Name() string { + return "trackingFilterPlugin" +} + +func (t *trackingFilterPlugin) Filter(ctx context.Context, state fwk.CycleState, pod *v1.Pod, nodeInfo fwk.NodeInfo) *fwk.Status { + t.lock.Lock() + defer t.lock.Unlock() + t.scoredNodes.Insert(nodeInfo.Node().Name) + return nil +} + +type scheduledPodPlacementFeasiblePlugin struct{} + +var _ fwk.PlacementFeasiblePlugin = &scheduledPodPlacementFeasiblePlugin{} + +func (p *scheduledPodPlacementFeasiblePlugin) Name() string { + return names.GangScheduling +} + +func (p *scheduledPodPlacementFeasiblePlugin) PlacementFeasible(_ context.Context, _ fwk.PlacementCycleState, _ fwk.PodGroupInfo, progress fwk.PlacementProgress) *fwk.Status { + if progress.Remaining > 0 { + return fwk.NewStatus(fwk.Wait) + } + if progress.Scheduled == 0 { + return fwk.NewStatus(fwk.Unschedulable, "no pods scheduled") + } + return nil +} + +func TestPodGroupSchedulingPlacementAlgorithm_PlacementLimit(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.TopologyAwareWorkloadScheduling: true, + features.GenericWorkload: true, + }) + + tests := []struct { + name string + numFeasiblePlacements int + numInfeasiblePlacements int + percentageLimit int32 + skipScore bool + expectedNumScoredPlacements int + expectedNumCheckedPlacements *int // nil if cannot be determined due to random placement order + }{ + { + name: "Only limits the feasible placements", + numFeasiblePlacements: 3, + numInfeasiblePlacements: 97, + percentageLimit: 2, + expectedNumScoredPlacements: 2, + }, + { + name: "Checks all placements in case there is no feasible placement", + numFeasiblePlacements: 0, + numInfeasiblePlacements: 100, + percentageLimit: 40, + expectedNumScoredPlacements: 0, + expectedNumCheckedPlacements: new(100), + }, + { + name: "Does not score placements if limit is 1", + numFeasiblePlacements: 100, + numInfeasiblePlacements: 0, + percentageLimit: 1, + expectedNumScoredPlacements: 0, + expectedNumCheckedPlacements: new(1), + }, + { + name: "Stops after a single placement if there are no score plugins", + numFeasiblePlacements: 100, + numInfeasiblePlacements: 0, + percentageLimit: 40, + skipScore: true, + expectedNumCheckedPlacements: new(1), + }, + } + + for _, test := range tests { + for _, compositePodGroup := range []bool{false, true} { + t.Run(fmt.Sprintf("%s (CPG: %v)", test.name, compositePodGroup), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CompositePodGroup, compositePodGroup) + logger, ctx := ktesting.NewTestContext(t) + + informerFactory := informers.NewSharedInformerFactory(clientsetfake.NewClientset(), 0) + queue := internalqueue.NewSchedulingQueue(nil, informerFactory) + + numNodes := test.numFeasiblePlacements + test.numInfeasiblePlacements + nodes := make([]*v1.Node, numNodes) + nodeNames := make([]string, numNodes) + + podGroupKey := fwk.PodGroupKey("default", "pg") + // The limit applies to the placements of the root group. A PodGroup under a + // CompositePodGroup gets a single placement within each root placement. + rootKey := podGroupKey + if compositePodGroup { + rootKey = fwk.CompositePodGroupKey("default", "cpg") + } + placementPlugin := &fakePlacementPlugin{ + name: "fakePlacementPlugin", + generatePlacementsResult: map[fwk.EntityKey]map[string][]string{ + rootKey: {}, + }, + filterStatus: make(map[string]*fwk.Status), + } + + for i := range numNodes { + nodeName := fmt.Sprintf("n%v", i) + nodes[i] = st.MakeNode().Name(nodeName).UID(nodeName).Obj() + nodeNames[i] = nodeName + placementName := fmt.Sprintf("p%v", i) + placementPlugin.generatePlacementsResult[rootKey][placementName] = []string{nodeName} + placementPlugin.filterStatus[nodeName] = fwk.NewStatus(fwk.Unschedulable) + } + if compositePodGroup { + placementPlugin.generatePlacementsResult[podGroupKey] = map[string][]string{"pg": nodeNames} + } + + feasiblePlacements := sets.New[string]() + for placement, nodeNames := range placementPlugin.generatePlacementsResult[rootKey] { + if len(feasiblePlacements) == test.numFeasiblePlacements { + break + } + placementPlugin.filterStatus[nodeNames[0]] = fwk.NewStatus(fwk.Success) + feasiblePlacements.Insert(placement) + } + + trackingFilter := &trackingFilterPlugin{scoredNodes: sets.New[string]()} + trackingScore := &trackingPlacementScorePlugin{scoredPlacements: sets.New[string]()} + placementFeasible := &scheduledPodPlacementFeasiblePlugin{} + + registry := []tf.RegisterPluginFunc{ + tf.RegisterPlacementGeneratePlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return placementPlugin, nil + }), + tf.RegisterFilterPlugin(trackingFilter.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return trackingFilter, nil + }), + tf.RegisterFilterPlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return placementPlugin, nil + }), + tf.RegisterPlacementFeasiblePlugin(placementFeasible.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return placementFeasible, nil + }), + } + + if !test.skipScore { + registry = append(registry, + tf.RegisterPlacementScorePlugin(trackingScore.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return trackingScore, nil + }, 1), + ) + } + + cache := internalcache.New(ctx, nil, true, compositePodGroup) + for _, node := range nodes { + cache.AddNode(logger, node) + } + testPodGroup := st.MakePodGroup().Name("pg").Namespace("default").Obj() + testCompositePodGroup := st.MakeCompositePodGroup().Name("cpg").Namespace("default").Obj() + cache.AddGenericPodGroup(fwk.NewGenericPodGroup(testPodGroup)) + if compositePodGroup { + cache.AddGenericPodGroup(fwk.NewGenericCompositePodGroup(testCompositePodGroup)) + } + snapshot := internalcache.NewEmptySnapshot() + if err := cache.UpdateSnapshot(logger, snapshot); err != nil { + t.Fatalf("Failed to update snapshot: %v", err) + } + + schedFwk, err := tf.NewFramework(ctx, + append(registry, + tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + ), + "test-scheduler", + frameworkruntime.WithInformerFactory(informerFactory), + frameworkruntime.WithSnapshotSharedLister(snapshot), + frameworkruntime.WithPodNominator(queue), + ) + if err != nil { + t.Fatalf("Failed to create new framework: %v", err) + } + + sched := &Scheduler{ + Cache: cache, + nodeInfoSnapshot: snapshot, + SchedulingQueue: queue, + Profiles: profile.Map{"test-scheduler": schedFwk}, + percentageOfPlacementsToScore: test.percentageLimit, + } + initTestAlgorithm(t, sched) + + podGroupPod := st.MakePod().Name("foo").Namespace("default").UID("foo").PodGroupName("pg").Obj() + runPlacementAlgorithm(ctx, t, sched, schedFwk, testPodGroup, testCompositePodGroup, compositePodGroup, podGroupPod) + + if got := len(trackingScore.scoredPlacements); got != test.expectedNumScoredPlacements { + t.Errorf("Unexpected number of scored placements, want %d, got %d", test.expectedNumScoredPlacements, got) + } + + if len(trackingScore.scoredPlacements) > 0 { + for skippedPlacement := range feasiblePlacements.Difference(trackingScore.scoredPlacements) { + placementNode := placementPlugin.generatePlacementsResult[rootKey][skippedPlacement][0] + if trackingFilter.scoredNodes.Has(placementNode) { + t.Errorf("Unexpected checked node for skipped placement %s, want none, got %v", skippedPlacement, placementNode) + } + } + } + + if test.expectedNumCheckedPlacements != nil { + // 1 unique node per placement + if got := len(trackingFilter.scoredNodes); got != *test.expectedNumCheckedPlacements { + t.Errorf("Unexpected number of checked placements, want %d, got %d", *test.expectedNumCheckedPlacements, got) + } + } + }) + } + } +} + +// runPlacementAlgorithm runs the placement algorithm for podGroup, or for a CompositePodGroup with +// podGroup as its only child when compositePodGroup is true, and reverts the assumed result. +func runPlacementAlgorithm(ctx context.Context, t *testing.T, sched *Scheduler, schedFwk framework.Framework, podGroup *schedulingv1beta1.PodGroup, cpg *schedulingv1alpha3.CompositePodGroup, compositePodGroup bool, pod *v1.Pod) *podGroupAlgorithmResult { + t.Helper() + podGroupInfo := &framework.PodGroupInfo{GenericPodGroup: fwk.NewGenericPodGroup(podGroup)} + podInfo := &framework.QueuedPodInfo{PodInfo: &framework.PodInfo{Pod: pod}} + var result *podGroupAlgorithmResult + var revert revertFns + if compositePodGroup { + cpgInfo := newQueuedPodGroupInfo(&framework.PodGroupInfo{ + GenericPodGroup: fwk.NewGenericCompositePodGroup(cpg), + Children: []*framework.PodGroupInfo{podGroupInfo}, + }, podInfo) + result, revert = sched.compositePodGroupSchedulingPlacementAlgorithm(ctx, schedFwk, framework.NewCycleState(), cpgInfo, cpgInfo.PodGroupInfo, map[fwk.EntityKey]*podGroupAlgorithmResult{}) + } else { + pgInfo := newQueuedPodGroupInfo(podGroupInfo, podInfo) + result, revert = sched.podGroupSchedulingPlacementAlgorithm(ctx, schedFwk, framework.NewCycleState(), pgInfo.PodGroupInfo, pgInfo) + } + revert.revert() + return result +} + +// fixedOrderPlacementPlugin returns one placement per node of the parent placement in a +// deterministic order, so a test can verify that the scheduler itself randomizes the evaluation +// order. It records the first slice it returns to verify that the scheduler doesn't mutate it. +type fixedOrderPlacementPlugin struct { + placementNames []string + nodePerPlacement map[string]string + firstPlacements []*fwk.Placement +} + +func (p *fixedOrderPlacementPlugin) Name() string { return "fixedOrderPlacementPlugin" } + +func (p *fixedOrderPlacementPlugin) GeneratePlacements(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, parentPlacement *fwk.Placement) (*fwk.GeneratePlacementsResult, *fwk.Status) { + parentNodes := map[string]fwk.NodeInfo{} + for _, node := range parentPlacement.Nodes { + parentNodes[node.Node().Name] = node + } + placements := make([]*fwk.Placement, 0, len(p.placementNames)) + for _, name := range p.placementNames { + if node, ok := parentNodes[p.nodePerPlacement[name]]; ok { + placements = append(placements, &fwk.Placement{ + Name: name, + Nodes: []fwk.NodeInfo{node}, + }) + } + } + if p.firstPlacements == nil { + p.firstPlacements = placements + } + return &fwk.GeneratePlacementsResult{Placements: placements}, nil +} + +func TestPodGroupSchedulingPlacementAlgorithm_UsesShufflePlacements(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.TopologyAwareWorkloadScheduling: true, + features.GenericWorkload: true, + }) + + const numPlacements = 10 + // The test shuffler moves this placement to the front. + const shuffledFirstPlacement = "p7" + allNodes := sets.New[string]() + for i := range numPlacements { + allNodes.Insert(fmt.Sprintf("n%v", i)) + } + + tests := []struct { + name string + compositePodGroup bool + percentageLimit int32 + nominatedNodeName string + wantCheckedNodes sets.Set[string] + wantShuffled bool + }{ + { + name: "first shuffled placement", + percentageLimit: 10, + wantCheckedNodes: sets.New("n7"), + wantShuffled: true, + }, + { + name: "nominated placement is used without shuffling", + percentageLimit: 10, + nominatedNodeName: "n0", + wantCheckedNodes: sets.New("n0"), + }, + { + name: "no shuffling when every placement may be scored", + percentageLimit: 100, + wantCheckedNodes: allNodes, + }, + { + name: "first shuffled placement of a CompositePodGroup", + compositePodGroup: true, + percentageLimit: 10, + wantCheckedNodes: sets.New("n7"), + wantShuffled: true, + }, + { + name: "no shuffling of CompositePodGroup placements when every placement may be scored", + compositePodGroup: true, + percentageLimit: 100, + wantCheckedNodes: allNodes, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CompositePodGroup, test.compositePodGroup) + logger, ctx := ktesting.NewTestContext(t) + + placementPlugin := &fixedOrderPlacementPlugin{nodePerPlacement: map[string]string{}} + nodes := make([]*v1.Node, numPlacements) + for i := range numPlacements { + nodeName := fmt.Sprintf("n%v", i) + placementName := fmt.Sprintf("p%v", i) + nodes[i] = st.MakeNode().Name(nodeName).UID(nodeName).Obj() + placementPlugin.placementNames = append(placementPlugin.placementNames, placementName) + placementPlugin.nodePerPlacement[placementName] = nodeName + } + + informerFactory := informers.NewSharedInformerFactory(clientsetfake.NewClientset(), 0) + queue := internalqueue.NewSchedulingQueue(nil, informerFactory) + + cache := internalcache.New(ctx, nil, true, test.compositePodGroup) + for _, node := range nodes { + cache.AddNode(logger, node) + } + testPodGroup := st.MakePodGroup().Name("pg").Namespace("default").Obj() + testCompositePodGroup := st.MakeCompositePodGroup().Name("cpg").Namespace("default").Obj() + cache.AddGenericPodGroup(fwk.NewGenericPodGroup(testPodGroup)) + if test.compositePodGroup { + cache.AddGenericPodGroup(fwk.NewGenericCompositePodGroup(testCompositePodGroup)) + } + snapshot := internalcache.NewEmptySnapshot() + if err := cache.UpdateSnapshot(logger, snapshot); err != nil { + t.Fatalf("Failed to update snapshot: %v", err) + } + + trackingFilter := &trackingFilterPlugin{scoredNodes: sets.New[string]()} + trackingScore := &trackingPlacementScorePlugin{scoredPlacements: sets.New[string]()} + registry := []tf.RegisterPluginFunc{ + tf.RegisterPlacementGeneratePlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return placementPlugin, nil + }), + tf.RegisterFilterPlugin(trackingFilter.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return trackingFilter, nil + }), + tf.RegisterPlacementScorePlugin(trackingScore.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return trackingScore, nil + }, 1), + tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + } + + schedFwk, err := tf.NewFramework(ctx, registry, "test-scheduler", + frameworkruntime.WithInformerFactory(informerFactory), + frameworkruntime.WithSnapshotSharedLister(snapshot), + frameworkruntime.WithPodNominator(queue), + ) + if err != nil { + t.Fatalf("Failed to create new framework: %v", err) + } + + shuffled := false + sched := &Scheduler{ + Cache: cache, + nodeInfoSnapshot: snapshot, + SchedulingQueue: queue, + Profiles: profile.Map{"test-scheduler": schedFwk}, + percentageOfPlacementsToScore: test.percentageLimit, + shufflePlacements: func(placements []*fwk.Placement) { + shuffled = true + for i, p := range placements { + if p.Name == shuffledFirstPlacement { + placements[0], placements[i] = placements[i], placements[0] + return + } + } + }, + } + initTestAlgorithm(t, sched) + + podGroupPod := st.MakePod().Name("foo").Namespace("default").UID("foo").PodGroupName("pg").NominatedNodeName(test.nominatedNodeName).Obj() + result := runPlacementAlgorithm(ctx, t, sched, schedFwk, testPodGroup, testCompositePodGroup, test.compositePodGroup, podGroupPod) + if !result.status.IsSuccess() { + t.Fatalf("Expected successful placement, got %v", result.status) + } + + if diff := cmp.Diff(sets.List(test.wantCheckedNodes), sets.List(trackingFilter.scoredNodes)); diff != "" { + t.Errorf("Unexpected checked nodes (-want,+got):\n%s", diff) + } + if shuffled != test.wantShuffled { + t.Errorf("Unexpected shuffle call, want %v, got %v", test.wantShuffled, shuffled) + } + var gotPluginOrder []string + for _, p := range placementPlugin.firstPlacements { + gotPluginOrder = append(gotPluginOrder, p.Name) + } + if diff := cmp.Diff(placementPlugin.placementNames, gotPluginOrder); diff != "" { + t.Errorf("Placements returned by the plugin were mutated (-want,+got):\n%s", diff) + } + }) + } +} + +func TestNewShufflePlacementsRandomizesOrder(t *testing.T) { + client := clientsetfake.NewClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sched, err := New(ctx, client, informerFactory, nil, profile.NewRecorderFactory(eventBroadcaster)) + if err != nil { + t.Fatalf("Failed to create scheduler: %v", err) + } + if sched.shufflePlacements == nil { + t.Fatal("Expected New to set a non-nil shufflePlacements function") + } + + const numPlacements = 10 + const runs = 50 + firstNames := sets.New[string]() + for range runs { + placements := make([]*fwk.Placement, numPlacements) + for i := range placements { + placements[i] = &fwk.Placement{Name: fmt.Sprintf("p%d", i)} + } + sched.shufflePlacements(placements) + firstNames.Insert(placements[0].Name) + } + + if len(firstNames) <= 1 { + t.Errorf("Expected the default shuffler to randomize placement order, but saw only %d distinct first placement(s) over %d runs", len(firstNames), runs) + } +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 688f2110b32..cf0914931ba 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -98,6 +98,12 @@ type Scheduler struct { nodeInfoSnapshot *internalcache.Snapshot + percentageOfPlacementsToScore int32 + + // shufflePlacements randomizes candidate placement order so the placement limit picks a fair + // subset. Set via New; nil disables shuffling to keep tests deterministic. + shufflePlacements func(placements []*fwk.Placement) + // logger *must* be initialized when creating a Scheduler, // otherwise logging functions will access a nil sink and // panic. @@ -127,6 +133,7 @@ type schedulerOptions struct { kubeConfig *restclient.Config // Overridden by profile level percentageOfNodesToScore if set in v1. percentageOfNodesToScore int32 + percentageOfPlacementsToScore int32 podInitialBackoffSeconds int64 podMaxBackoffSeconds int64 podMaxInUnschedulablePodsDuration time.Duration @@ -199,6 +206,16 @@ func WithPercentageOfNodesToScore(percentageOfNodesToScore *int32) Option { } } +// WithPercentageOfPlacementsToScore sets percentageOfPlacementsToScore for Scheduler. +// The default value of 0 will use an adaptive percentage. +func WithPercentageOfPlacementsToScore(percentageOfPlacementsToScore *int32) Option { + return func(o *schedulerOptions) { + if percentageOfPlacementsToScore != nil { + o.percentageOfPlacementsToScore = *percentageOfPlacementsToScore + } + } +} + // WithFrameworkOutOfTreeRegistry sets the registry for out-of-tree plugins. Those plugins // will be appended to the default registry. func WithFrameworkOutOfTreeRegistry(registry frameworkruntime.Registry) Option { @@ -262,6 +279,7 @@ func WithBuildFrameworkCapturer(fc FrameworkCapturer) Option { var defaultSchedulerOptions = schedulerOptions{ clock: clock.RealClock{}, percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, + percentageOfPlacementsToScore: schedulerapi.DefaultPercentageOfPlacementsToScore, podInitialBackoffSeconds: int64(internalqueue.DefaultPodInitialBackoffDuration.Seconds()), podMaxBackoffSeconds: int64(internalqueue.DefaultPodMaxBackoffDuration.Seconds()), podMaxInUnschedulablePodsDuration: internalqueue.DefaultPodMaxInUnschedulablePodsDuration, @@ -374,6 +392,8 @@ func New(ctx context.Context, Cache: schedulerCache, client: client, nodeInfoSnapshot: snapshot, + percentageOfPlacementsToScore: options.percentageOfPlacementsToScore, + shufflePlacements: randShufflePlacements, StopEverything: stopEverything, SchedulingQueue: podQueue, Profiles: profiles, diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index d48e52e9024..4ea1336cb19 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -470,6 +470,51 @@ func TestWithPercentageOfNodesToScore(t *testing.T) { } } +// TestWithPercentageOfPlacementsToScore tests scheduler's PercentageOfPlacementsToScore is set correctly. +func TestWithPercentageOfPlacementsToScore(t *testing.T) { + tests := []struct { + name string + percentageOfPlacementsToScoreConfig *int32 + wantedPercentageOfPlacementsToScore int32 + }{ + { + name: "percentageOfPlacementsScore is nil", + percentageOfPlacementsToScoreConfig: nil, + wantedPercentageOfPlacementsToScore: schedulerapi.DefaultPercentageOfPlacementsToScore, + }, + { + name: "percentageOfPlacementsScore is not nil", + percentageOfPlacementsToScoreConfig: ptr.To[int32](10), + wantedPercentageOfPlacementsToScore: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) + _, ctx := ktesting.NewTestContext(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + sched, err := New( + ctx, + client, + informerFactory, + nil, + profile.NewRecorderFactory(eventBroadcaster), + WithPercentageOfPlacementsToScore(tt.percentageOfPlacementsToScoreConfig), + ) + if err != nil { + t.Fatalf("Failed to create scheduler: %v", err) + } + if sched.percentageOfPlacementsToScore != tt.wantedPercentageOfPlacementsToScore { + t.Errorf("scheduler.percentageOfPlacementsToScore = %v, want %v", sched.percentageOfPlacementsToScore, tt.wantedPercentageOfPlacementsToScore) + } + }) + } +} + // getPodFromPriorityQueue is the function used in the TestDefaultErrorFunc test to get // the specific pod from the given priority queue. It returns the found pod in the priority queue. func getPodFromPriorityQueue(queue *internalqueue.PriorityQueue, pod *v1.Pod) *v1.Pod { diff --git a/staging/src/k8s.io/kube-scheduler/config/v1/types.go b/staging/src/k8s.io/kube-scheduler/config/v1/types.go index c4c7fd15076..c97d1a0f9fb 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1/types.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1/types.go @@ -68,6 +68,23 @@ type KubeSchedulerConfiguration struct { // nodes will be scored. It is overridden by profile level PercentageOfNodesToScore. PercentageOfNodesToScore *int32 `json:"percentageOfNodesToScore,omitempty"` + // PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler + // tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and + // scores them. Valid values are from 0 to 100. + // The scheduler always tries to find at least one feasible placement. + // Example: if 500 placements are generated and the value is 30, the scheduler stops after + // finding 150 feasible placements. + // When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node + // counts across all generated placements. If placements overlap, a node present in multiple + // placements is counted once per placement. The adaptive percentage decreases linearly from + // 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger + // sums down to a minimum of 5%. + // It is overridden by profile level PercentageOfPlacementsToScore. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + PercentageOfPlacementsToScore *int32 `json:"percentageOfPlacementsToScore,omitempty"` + // PodInitialBackoffSeconds is the initial backoff for unschedulable pods. // If specified, it must be greater than 0. If this value is null, the default value (1s) // will be used. @@ -153,6 +170,24 @@ type KubeSchedulerProfile struct { // global PercentageOfNodesToScore will be used. PercentageOfNodesToScore *int32 `json:"percentageOfNodesToScore,omitempty"` + // PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler + // tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and + // scores them. Valid values are from 0 to 100. + // The scheduler always tries to find at least one feasible placement. + // Example: if 500 placements are generated and the value is 30, the scheduler stops after + // finding 150 feasible placements. + // When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node + // counts across all generated placements. If placements overlap, a node present in multiple + // placements is counted once per placement. The adaptive percentage decreases linearly from + // 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger + // sums down to a minimum of 5%. + // It overrides global PercentageOfPlacementsToScore. If empty, the global + // PercentageOfPlacementsToScore is used. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + PercentageOfPlacementsToScore *int32 `json:"percentageOfPlacementsToScore,omitempty"` + // Plugins specify the set of plugins that should be enabled or disabled. // Enabled plugins are the ones that should be enabled in addition to the // default plugins. Disabled plugins are any of the default plugins that diff --git a/staging/src/k8s.io/kube-scheduler/config/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-scheduler/config/v1/zz_generated.deepcopy.go index b4b2e5c9a3e..9c7d0a9a53a 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1/zz_generated.deepcopy.go @@ -218,6 +218,11 @@ func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfigurati *out = new(int32) **out = **in } + if in.PercentageOfPlacementsToScore != nil { + in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore + *out = new(int32) + **out = **in + } if in.PodInitialBackoffSeconds != nil { in, out := &in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds *out = new(int64) @@ -276,6 +281,11 @@ func (in *KubeSchedulerProfile) DeepCopyInto(out *KubeSchedulerProfile) { *out = new(int32) **out = **in } + if in.PercentageOfPlacementsToScore != nil { + in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore + *out = new(int32) + **out = **in + } if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = new(Plugins) diff --git a/staging/src/k8s.io/kube-scheduler/framework/interface.go b/staging/src/k8s.io/kube-scheduler/framework/interface.go index 3dfac5e1121..8469e9f81a0 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/interface.go +++ b/staging/src/k8s.io/kube-scheduler/framework/interface.go @@ -807,6 +807,8 @@ type PlacementGeneratePlugin interface { // GeneratePlacements generates a list of potential Placements for the given PodGroup within the parent placement. // Each Placement represents a candidate set of resources, e.g., nodes matching a selector. + // The scheduler may shuffle and evaluate only a subset of the returned Placements, so plugins + // must not rely on their order. GeneratePlacements(ctx context.Context, state PodGroupCycleState, podGroup PodGroupInfo, parentPlacement *Placement) (*GeneratePlacementsResult, *Status) } diff --git a/test/integration/scheduler/podgroup/topology_aware_scheduling/cpg_tas_test.go b/test/integration/scheduler/podgroup/topology_aware_scheduling/cpg_tas_test.go index 886abdde68d..1005e2376d4 100644 --- a/test/integration/scheduler/podgroup/topology_aware_scheduling/cpg_tas_test.go +++ b/test/integration/scheduler/podgroup/topology_aware_scheduling/cpg_tas_test.go @@ -2008,16 +2008,200 @@ func TestCPGTopologyAwareSchedulingWorkloadAwarePreemption(t *testing.T) { } } -func runCPGTestScenario(t *testing.T, tt scenario) { +func TestCPGTopologyAwareSchedulingWithPlacementLimit(t *testing.T) { + tests := []scenario{ + { + name: "parent CPG co-locates children on a single rack despite a restrictive placement limit", + steps: []stepsframework.Step{ + { + Name: "Create nodes in multiple racks, each rack able to fit the whole hierarchy", + CreateNodes: []*v1.Node{ + makeNode("node1-z1-r1", "rack-1", "zone-1"), + makeNode("node2-z1-r1", "rack-1", "zone-1"), + makeNode("node3-z1-r2", "rack-2", "zone-1"), + makeNode("node4-z1-r2", "rack-2", "zone-1"), + makeNode("node5-z2-r3", "rack-3", "zone-2"), + makeNode("node6-z2-r3", "rack-3", "zone-2"), + }, + }, + { + Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=rack)", + CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "rack", 2), + }, + { + Name: "Create child PodGroup pg1 (Gang with minCount=2, without topology constraints, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "", 2), + }, + { + Name: "Create child PodGroup pg2 (Gang with minCount=2, without topology constraints, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "", 2), + }, + { + Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU", + CreatePods: []*v1.Pod{ + makePod("p1", "pg1"), + makePod("p2", "pg1"), + makePod("p3", "pg2"), + makePod("p4", "pg2"), + }, + }, + { + Name: "Verify all pods in the composite group are scheduled", + WaitForPodsScheduled: []string{"p1", "p2", "p3", "p4"}, + }, + { + Name: "Verify all pods across both children landed on a single rack despite the limit", + VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{ + Pods: []string{"p1", "p2", "p3", "p4"}, + TopologyKey: "rack", + }, + }, + }, + }, + { + name: "restrictive placement limit still finds the only feasible rack for a CPG", + steps: []stepsframework.Step{ + { + Name: "Create nodes in multiple racks; only rack-1 can fit the whole hierarchy", + CreateNodes: []*v1.Node{ + makeNode("node1-z1-r1", "rack-1", "zone-1"), + makeNode("node2-z1-r1", "rack-1", "zone-1"), + makeNode("node3-z1-r1", "rack-1", "zone-1"), + makeNode("node4-z1-r2", "rack-2", "zone-1"), + makeNode("node5-z1-r2", "rack-2", "zone-1"), + makeNode("node6-z2-r3", "rack-3", "zone-2"), + }, + }, + { + Name: "Block rack-2 so it can no longer fit the whole hierarchy", + CreatePods: []*v1.Pod{ + makeAssignedPod("existing1", "node4-z1-r2", "2"), + }, + }, + { + Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=rack)", + CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "rack", 2), + }, + { + Name: "Create child PodGroup pg1 (Gang with minCount=2, without topology constraints, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "", 2), + }, + { + Name: "Create child PodGroup pg2 (Gang with minCount=1, without topology constraints, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "", 1), + }, + { + Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU", + CreatePods: []*v1.Pod{ + makePod("p1", "pg1"), + makePod("p2", "pg1"), + makePod("p3", "pg2"), + }, + }, + { + Name: "Verify all pods in the composite group are scheduled", + WaitForPodsScheduled: []string{"p1", "p2", "p3"}, + }, + { + Name: "Verify all pods landed on the only feasible rack", + VerifyAssignments: &stepsframework.VerifyAssignments{ + Pods: []string{"p1", "p2", "p3"}, + Nodes: sets.New("node1-z1-r1", "node2-z1-r1", "node3-z1-r1"), + }, + }, + }, + }, + { + name: "restrictive placement limit applies at each level of a CPG hierarchy (root=zone, leaf=rack)", + steps: []stepsframework.Step{ + { + Name: "Create nodes across zones and racks", + CreateNodes: []*v1.Node{ + makeNode("node1-z1-r1", "rack-1", "zone-1"), + makeNode("node2-z1-r1", "rack-1", "zone-1"), + makeNode("node3-z1-r2", "rack-2", "zone-1"), + makeNode("node4-z1-r2", "rack-2", "zone-1"), + makeNode("node5-z2-r3", "rack-3", "zone-2"), + makeNode("node6-z2-r3", "rack-3", "zone-2"), + }, + }, + { + Name: "Block zone-2 so it can no longer fit the whole hierarchy", + CreatePods: []*v1.Pod{ + makeAssignedPod("existing1", "node6-z2-r3", "2"), + }, + }, + { + Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=zone)", + CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "zone", 2), + }, + { + Name: "Create child PodGroup pg1 (Gang with minCount=2, TopologyKey=rack, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "rack", 2), + }, + { + Name: "Create child PodGroup pg2 (Gang with minCount=2, TopologyKey=rack, Parent=cpg-root)", + CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "rack", 2), + }, + { + Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU", + CreatePods: []*v1.Pod{ + makePod("p1", "pg1"), + makePod("p2", "pg1"), + makePod("p3", "pg2"), + makePod("p4", "pg2"), + }, + }, + { + Name: "Verify all pods in the composite group are scheduled", + WaitForPodsScheduled: []string{"p1", "p2", "p3", "p4"}, + }, + { + Name: "Verify all pods landed on the only feasible zone", + VerifyAssignments: &stepsframework.VerifyAssignments{ + Pods: []string{"p1", "p2", "p3", "p4"}, + Nodes: sets.New("node1-z1-r1", "node2-z1-r1", "node3-z1-r2", "node4-z1-r2"), + }, + }, + { + Name: "Verify pg1 landed on a single rack", + VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{ + Pods: []string{"p1", "p2"}, + TopologyKey: "rack", + }, + }, + { + Name: "Verify pg2 landed on a single rack", + VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{ + Pods: []string{"p3", "p4"}, + TopologyKey: "rack", + }, + }, + }, + }, + } + + // A low percentage forces the scheduler to score only a subset of the generated placements. + percentageLimit := int32(1) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runCPGTestScenario(t, tt, scheduler.WithPercentageOfPlacementsToScore(new(percentageLimit))) + }) + } +} + +func runCPGTestScenario(t *testing.T, tt scenario, opts ...scheduler.Option) { featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.CompositePodGroup: true, features.GenericWorkload: true, features.TopologyAwareWorkloadScheduling: true, }) - testCtx := testutils.InitTestSchedulerWithNS(t, "cpg-tas", + opts = append([]scheduler.Option{ scheduler.WithPodMaxBackoffSeconds(0), - scheduler.WithPodInitialBackoffSeconds(0)) + scheduler.WithPodInitialBackoffSeconds(0), + }, opts...) + testCtx := testutils.InitTestSchedulerWithNS(t, "cpg-tas", opts...) ns := testCtx.NS.Name if err := stepsframework.RunSteps(testCtx, t, ns, tt.steps); err != nil { diff --git a/test/integration/scheduler/podgroup/topology_aware_scheduling/tas_test.go b/test/integration/scheduler/podgroup/topology_aware_scheduling/tas_test.go index 58589226ab3..60818c1b1ae 100644 --- a/test/integration/scheduler/podgroup/topology_aware_scheduling/tas_test.go +++ b/test/integration/scheduler/podgroup/topology_aware_scheduling/tas_test.go @@ -1752,3 +1752,125 @@ func runTestScenario(t *testing.T, tt scenario, cpgEnabled bool) { t.Fatal(err) } } + +func TestTopologyAwareSchedulingWithPlacementLimit(t *testing.T) { + tests := []struct { + name string + steps []stepsframework.Step + }{ + { + name: "basic podgroup co-locates on a single rack despite a restrictive placement limit", + steps: []stepsframework.Step{ + { + Name: "Create nodes in multiple racks, each rack able to fit the whole group", + CreateNodes: []*v1.Node{ + makeNode("node1-rack1", "rack-1", "zone-1"), + makeNode("node2-rack1", "rack-1", "zone-1"), + makeNode("node3-rack2", "rack-2", "zone-1"), + makeNode("node4-rack2", "rack-2", "zone-1"), + makeNode("node5-rack3", "rack-3", "zone-2"), + makeNode("node6-rack3", "rack-3", "zone-2"), + }, + }, + { + Name: "Create the PodGroup constrained to a single rack", + CreatePodGroup: makeBasicPodGroup("pg1", "rack"), + }, + { + Name: "Create all pods belonging to the podgroup", + CreatePods: []*v1.Pod{ + makePod("p1", "pg1"), + makePod("p2", "pg1"), + makePod("p3", "pg1"), + }, + }, + { + Name: "Verify the entire podgroup is scheduled", + WaitForPodsScheduled: []string{"p1", "p2", "p3"}, + }, + { + Name: "Verify all pods landed on a single rack despite the limit", + VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{ + Pods: []string{"p1", "p2", "p3"}, + TopologyKey: "rack", + }, + }, + }, + }, + { + name: "restrictive placement limit still finds the only feasible rack for a gang", + steps: []stepsframework.Step{ + { + Name: "Create nodes in multiple racks; only rack-1 can fit the whole gang", + CreateNodes: []*v1.Node{ + makeNode("node1-rack1", "rack-1", "zone-1"), + makeNode("node2-rack1", "rack-1", "zone-1"), + makeNode("node3-rack1", "rack-1", "zone-1"), + makeNode("node4-rack2", "rack-2", "zone-1"), + makeNode("node5-rack2", "rack-2", "zone-1"), + makeNode("node6-zone2", "rack-3", "zone-2"), + }, + }, + { + Name: "Block rack-2 so it can no longer fit the whole gang", + CreatePods: []*v1.Pod{ + makeAssignedPod("existing1", "node4-rack2", "2"), + }, + }, + { + Name: "Create the PodGroup object (Gang with minCount=3) constrained to a single rack", + CreatePodGroup: makeGangPodGroup("pg1", "rack", 3), + }, + { + Name: "Create all pods belonging to the gang", + CreatePods: []*v1.Pod{ + makePod("p1", "pg1"), + makePod("p2", "pg1"), + makePod("p3", "pg1"), + }, + }, + { + Name: "Verify the entire gang is scheduled", + WaitForPodsScheduled: []string{"p1", "p2", "p3"}, + }, + { + Name: "Verify all pods landed on the only feasible rack", + VerifyAssignments: &stepsframework.VerifyAssignments{ + Pods: []string{"p1", "p2", "p3"}, + Nodes: sets.New("node1-rack1", "node2-rack1", "node3-rack1"), + }, + }, + }, + }, + } + + // A low percentage forces the scheduler to score only a subset of the generated placements. + percentageLimit := int32(1) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }) + + testCtx := testutils.InitTestSchedulerWithNS(t, "tas-placement-limit", + scheduler.WithPodMaxBackoffSeconds(0), + scheduler.WithPodInitialBackoffSeconds(0), + scheduler.WithPercentageOfPlacementsToScore(new(percentageLimit))) + ns := testCtx.NS.Name + + workload := st.MakeWorkload().Name("workload").Namespace(ns). + PodGroupTemplate(st.MakePodGroupTemplate().Name("t1").MinCount(1).Obj()). + Obj() + workloadStep := []stepsframework.Step{ + { + Name: "Creating workload", + CreateWorkloads: []*schedulingapi.Workload{workload}, + }, + } + if err := stepsframework.RunSteps(testCtx, t, ns, append(workloadStep, tt.steps...)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 0d1a9ebe1da..fc4bf51180f 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -117,6 +117,7 @@ func StartSchedulerWithDone(tCtx ktesting.TContext, cfg *kubeschedulerconfig.Kub scheduler.WithKubeConfig(tCtx.RESTConfig()), scheduler.WithProfiles(cfg.Profiles...), scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore), + scheduler.WithPercentageOfPlacementsToScore(cfg.PercentageOfPlacementsToScore), scheduler.WithPodMaxBackoffSeconds(cfg.PodMaxBackoffSeconds), scheduler.WithPodInitialBackoffSeconds(cfg.PodInitialBackoffSeconds), scheduler.WithExtenders(cfg.Extenders...),