mirror of
https://github.com/kubernetes/kubernetes.git
synced 2026-10-28 13:45:38 +08:00
Introduce PercentageOfPlacementsToScore limit for TAS (#139859)
* Introduce PercentageOfPlacementsToScore limit for TAS Co-authored-by: Bartosz <brejman@google.com> * Shuffle candidate placements to guarantee fair evaluation order * Add integration tests for TAS placement limit * Reuse percentage validation for scheduler scoring limits Signed-off-by: William Yao <william2000yao@gmail.com> * Fix scheduler tests after rebase on upstream master Drop the DynamicResources defaults case, whose feature gate no longer exists upstream, and build QueuedPodGroupInfo through the existing newQueuedPodGroupInfo helper now that queuedPodInfos is unexported. * Apply placement limit to CompositePodGroups Use the shared PercentageOfPlacementsToScore limit and placement shuffling for CompositePodGroup placements as well as PodGroup placements, and skip the limit and shuffle setup when the nominated placement is feasible. Clarify the field docs, exercise the real Filter path in the placement limit test, and cover shuffle skipping, plugin slice immutability and CompositePodGroups in unit and integration tests. * Reword DefaultPercentageOfPlacementsToScore godoc --------- Signed-off-by: William Yao <william2000yao@gmail.com> Co-authored-by: Bartosz <brejman@google.com>
This commit is contained in:
1 parent
f3a03cbd1d
commit
ccca46cf5f
23 files changed
+1462
-37
No files matched your search
@@ -470,6 +470,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
|
||||
scheduler.WithKubeConfig(cc.KubeConfig),
|
||||
scheduler.WithProfiles(cc.ComponentConfig.Profiles...),
|
||||
scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore),
|
||||
scheduler.WithPercentageOfPlacementsToScore(cc.ComponentConfig.PercentageOfPlacementsToScore),
|
||||
scheduler.WithFrameworkOutOfTreeRegistry(outOfTreeRegistry),
|
||||
scheduler.WithPodMaxBackoffSeconds(cc.ComponentConfig.PodMaxBackoffSeconds),
|
||||
scheduler.WithPodInitialBackoffSeconds(cc.ComponentConfig.PodInitialBackoffSeconds),
|
||||
|
||||
+14
@@ -73549,6 +73549,13 @@ func schema_k8sio_kube_scheduler_config_v1_KubeSchedulerConfiguration(ref common
|
||||
Format: "int32",
|
||||
},
|
||||
},
|
||||
"percentageOfPlacementsToScore": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and scores them. Valid values are from 0 to 100. The scheduler always tries to find at least one feasible placement. Example: if 500 placements are generated and the value is 30, the scheduler stops after finding 150 feasible placements. When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node counts across all generated placements. If placements overlap, a node present in multiple placements is counted once per placement. The adaptive percentage decreases linearly from 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger sums down to a minimum of 5%. It is overridden by profile level PercentageOfPlacementsToScore. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.",
|
||||
Type: []string{"integer"},
|
||||
Format: "int32",
|
||||
},
|
||||
},
|
||||
"podInitialBackoffSeconds": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "PodInitialBackoffSeconds is the initial backoff for unschedulable pods. If specified, it must be greater than 0. If this value is null, the default value (1s) will be used.",
|
||||
@@ -73639,6 +73646,13 @@ func schema_k8sio_kube_scheduler_config_v1_KubeSchedulerProfile(ref common.Refer
|
||||
Format: "int32",
|
||||
},
|
||||
},
|
||||
"percentageOfPlacementsToScore": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and scores them. Valid values are from 0 to 100. The scheduler always tries to find at least one feasible placement. Example: if 500 placements are generated and the value is 30, the scheduler stops after finding 150 feasible placements. When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node counts across all generated placements. If placements overlap, a node present in multiple placements is counted once per placement. The adaptive percentage decreases linearly from 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger sums down to a minimum of 5%. It overrides global PercentageOfPlacementsToScore. If empty, the global PercentageOfPlacementsToScore is used. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.",
|
||||
Type: []string{"integer"},
|
||||
Format: "int32",
|
||||
},
|
||||
},
|
||||
"plugins": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "Plugins specify the set of plugins that should be enabled or disabled. Enabled plugins are the ones that should be enabled in addition to the default plugins. Disabled plugins are any of the default plugins that should be disabled. When no enabled or disabled plugin is specified for an extension point, default plugins for that extension point will be used if there is any. If a QueueSort plugin is specified, the same QueueSort Plugin and PluginConfig must be specified for all profiles.",
|
||||
|
||||
@@ -219,6 +219,40 @@ profiles:
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 with non-default global percentageOfPlacementsToScore",
|
||||
data: []byte(`
|
||||
apiVersion: kubescheduler.config.k8s.io/v1
|
||||
kind: KubeSchedulerConfiguration
|
||||
percentageOfPlacementsToScore: 10
|
||||
`),
|
||||
wantProfiles: []config.KubeSchedulerProfile{
|
||||
{
|
||||
SchedulerName: "default-scheduler",
|
||||
PercentageOfPlacementsToScore: nil,
|
||||
Plugins: defaults.PluginsV1,
|
||||
PluginConfig: defaults.PluginConfigsV1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 with non-default global and profile percentageOfPlacementsToScore",
|
||||
data: []byte(`
|
||||
apiVersion: kubescheduler.config.k8s.io/v1
|
||||
kind: KubeSchedulerConfiguration
|
||||
percentageOfPlacementsToScore: 10
|
||||
profiles:
|
||||
- percentageOfPlacementsToScore: 20
|
||||
`),
|
||||
wantProfiles: []config.KubeSchedulerProfile{
|
||||
{
|
||||
SchedulerName: "default-scheduler",
|
||||
PercentageOfPlacementsToScore: ptr.To[int32](20),
|
||||
Plugins: defaults.PluginsV1,
|
||||
PluginConfig: defaults.PluginConfigsV1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 plugins can include version and kind",
|
||||
data: []byte(`
|
||||
|
||||
@@ -69,6 +69,23 @@ type KubeSchedulerConfiguration struct {
|
||||
// nodes will be scored. It is overridden by profile level PercentageOfNodesToScore.
|
||||
PercentageOfNodesToScore *int32
|
||||
|
||||
// PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler
|
||||
// tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and
|
||||
// scores them. Valid values are from 0 to 100.
|
||||
// The scheduler always tries to find at least one feasible placement.
|
||||
// Example: if 500 placements are generated and the value is 30, the scheduler stops after
|
||||
// finding 150 feasible placements.
|
||||
// When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node
|
||||
// counts across all generated placements. If placements overlap, a node present in multiple
|
||||
// placements is counted once per placement. The adaptive percentage decreases linearly from
|
||||
// 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger
|
||||
// sums down to a minimum of 5%.
|
||||
// It is overridden by profile level PercentageOfPlacementsToScore.
|
||||
// This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.
|
||||
// +featureGate=TopologyAwareWorkloadScheduling
|
||||
// +optional
|
||||
PercentageOfPlacementsToScore *int32
|
||||
|
||||
// PodInitialBackoffSeconds is the initial backoff for unschedulable pods.
|
||||
// If specified, it must be greater than 0. If this value is null, the default value (1s)
|
||||
// will be used.
|
||||
@@ -114,6 +131,24 @@ type KubeSchedulerProfile struct {
|
||||
// global PercentageOfNodesToScore will be used.
|
||||
PercentageOfNodesToScore *int32
|
||||
|
||||
// PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler
|
||||
// tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and
|
||||
// scores them. Valid values are from 0 to 100.
|
||||
// The scheduler always tries to find at least one feasible placement.
|
||||
// Example: if 500 placements are generated and the value is 30, the scheduler stops after
|
||||
// finding 150 feasible placements.
|
||||
// When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node
|
||||
// counts across all generated placements. If placements overlap, a node present in multiple
|
||||
// placements is counted once per placement. The adaptive percentage decreases linearly from
|
||||
// 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger
|
||||
// sums down to a minimum of 5%.
|
||||
// It overrides global PercentageOfPlacementsToScore. If empty, the global
|
||||
// PercentageOfPlacementsToScore is used.
|
||||
// This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.
|
||||
// +featureGate=TopologyAwareWorkloadScheduling
|
||||
// +optional
|
||||
PercentageOfPlacementsToScore *int32
|
||||
|
||||
// Plugins specify the set of plugins that should be enabled or disabled.
|
||||
// Enabled plugins are the ones that should be enabled in addition to the
|
||||
// default plugins. Disabled plugins are any of the default plugins that
|
||||
@@ -229,6 +264,11 @@ const (
|
||||
// A value of 0 means adaptive, meaning the scheduler figures out a proper default.
|
||||
DefaultPercentageOfNodesToScore = 0
|
||||
|
||||
// DefaultPercentageOfPlacementsToScore is the default percentage of generated placements that
|
||||
// the scheduler tries to find feasible before it stops searching.
|
||||
// A value of 0 means adaptive, meaning the scheduler figures out a proper default.
|
||||
DefaultPercentageOfPlacementsToScore = 0
|
||||
|
||||
// MaxCustomPriorityScore is the max score UtilizationShapePoint expects.
|
||||
MaxCustomPriorityScore int64 = 10
|
||||
|
||||
|
||||
@@ -132,6 +132,10 @@ func SetDefaults_KubeSchedulerConfiguration(obj *configv1.KubeSchedulerConfigura
|
||||
obj.PercentageOfNodesToScore = ptr.To[int32](config.DefaultPercentageOfNodesToScore)
|
||||
}
|
||||
|
||||
if obj.PercentageOfPlacementsToScore == nil && feature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) {
|
||||
obj.PercentageOfPlacementsToScore = ptr.To[int32](config.DefaultPercentageOfPlacementsToScore)
|
||||
}
|
||||
|
||||
if len(obj.LeaderElection.ResourceLock) == 0 {
|
||||
// Use lease-based leader election to reduce cost.
|
||||
// We migrated for EndpointsLease lock in 1.17 and starting in 1.20 we
|
||||
|
||||
@@ -675,11 +675,51 @@ func TestSchedulerDefaults(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "default TopologyAwareWorkloadScheduling",
|
||||
features: map[featuregate.Feature]bool{features.GenericWorkload: true, features.TopologyAwareWorkloadScheduling: true},
|
||||
config: &configv1.KubeSchedulerConfiguration{},
|
||||
expected: &configv1.KubeSchedulerConfiguration{
|
||||
Parallelism: ptr.To[int32](16),
|
||||
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
|
||||
EnableProfiling: &enable,
|
||||
EnableContentionProfiling: &enable,
|
||||
},
|
||||
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
|
||||
LeaderElect: new(true),
|
||||
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
|
||||
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
|
||||
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
|
||||
ResourceLock: "leases",
|
||||
ResourceNamespace: "kube-system",
|
||||
ResourceName: "kube-scheduler",
|
||||
},
|
||||
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
|
||||
QPS: 50,
|
||||
Burst: 100,
|
||||
ContentType: "application/vnd.kubernetes.protobuf",
|
||||
},
|
||||
PercentageOfNodesToScore: ptr.To[int32](config.DefaultPercentageOfNodesToScore),
|
||||
PercentageOfPlacementsToScore: ptr.To[int32](config.DefaultPercentageOfPlacementsToScore),
|
||||
PodInitialBackoffSeconds: ptr.To[int64](1),
|
||||
PodMaxBackoffSeconds: ptr.To[int64](10),
|
||||
Profiles: []configv1.KubeSchedulerProfile{
|
||||
{
|
||||
PluginConfig: pluginConfigs,
|
||||
SchedulerName: new("default-scheduler"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, feature.DefaultFeatureGate, tc.features)
|
||||
if len(tc.features) > 0 && len(tc.expected.Profiles) > 0 {
|
||||
// Recompute plugin defaults after enabling feature gates for this test.
|
||||
tc.expected.Profiles[0].Plugins = getDefaultPlugins()
|
||||
}
|
||||
SetDefaults_KubeSchedulerConfiguration(tc.config)
|
||||
if diff := cmp.Diff(tc.expected, tc.config); diff != "" {
|
||||
t.Errorf("Got unexpected defaults (-want, +got):\n%s", diff)
|
||||
|
||||
@@ -419,6 +419,7 @@ func autoConvert_v1_KubeSchedulerConfiguration_To_config_KubeSchedulerConfigurat
|
||||
return err
|
||||
}
|
||||
out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore))
|
||||
out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore))
|
||||
if err := metav1.Convert_Pointer_int64_To_int64(&in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds, s); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -455,6 +456,7 @@ func autoConvert_config_KubeSchedulerConfiguration_To_v1_KubeSchedulerConfigurat
|
||||
return err
|
||||
}
|
||||
out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore))
|
||||
out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore))
|
||||
if err := metav1.Convert_int64_To_Pointer_int64(&in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds, s); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -482,6 +484,7 @@ func autoConvert_v1_KubeSchedulerProfile_To_config_KubeSchedulerProfile(in *conf
|
||||
return err
|
||||
}
|
||||
out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore))
|
||||
out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore))
|
||||
if in.Plugins != nil {
|
||||
in, out := &in.Plugins, &out.Plugins
|
||||
*out = new(config.Plugins)
|
||||
@@ -515,6 +518,7 @@ func autoConvert_config_KubeSchedulerProfile_To_v1_KubeSchedulerProfile(in *conf
|
||||
return err
|
||||
}
|
||||
out.PercentageOfNodesToScore = (*int32)(unsafe.Pointer(in.PercentageOfNodesToScore))
|
||||
out.PercentageOfPlacementsToScore = (*int32)(unsafe.Pointer(in.PercentageOfPlacementsToScore))
|
||||
if in.Plugins != nil {
|
||||
in, out := &in.Plugins, &out.Plugins
|
||||
*out = new(configv1.Plugins)
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
componentbasevalidation "k8s.io/component-base/config/validation"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
schedfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
|
||||
)
|
||||
@@ -68,7 +69,8 @@ func ValidateKubeSchedulerConfiguration(cc *config.KubeSchedulerConfiguration) u
|
||||
errs = append(errs, validateCommonQueueSort(profilesPath, cc.Profiles)...)
|
||||
}
|
||||
|
||||
errs = append(errs, validatePercentageOfNodesToScore(field.NewPath("percentageOfNodesToScore"), cc.PercentageOfNodesToScore))
|
||||
errs = append(errs, validatePercentage(field.NewPath("percentageOfNodesToScore"), cc.PercentageOfNodesToScore))
|
||||
errs = append(errs, validatePercentageOfPlacementsToScore(field.NewPath("percentageOfPlacementsToScore"), cc.PercentageOfPlacementsToScore))
|
||||
|
||||
if cc.PodInitialBackoffSeconds <= 0 {
|
||||
errs = append(errs, field.Invalid(field.NewPath("podInitialBackoffSeconds"),
|
||||
@@ -83,15 +85,24 @@ func ValidateKubeSchedulerConfiguration(cc *config.KubeSchedulerConfiguration) u
|
||||
return utilerrors.Flatten(utilerrors.NewAggregate(errs))
|
||||
}
|
||||
|
||||
func validatePercentageOfNodesToScore(path *field.Path, percentageOfNodesToScore *int32) error {
|
||||
if percentageOfNodesToScore != nil {
|
||||
if *percentageOfNodesToScore < 0 || *percentageOfNodesToScore > 100 {
|
||||
return field.Invalid(path, *percentageOfNodesToScore, "not in valid range [0-100]")
|
||||
func validatePercentage(path *field.Path, percentage *int32) error {
|
||||
if percentage != nil {
|
||||
if *percentage < 0 || *percentage > 100 {
|
||||
return field.Invalid(path, *percentage, "not in valid range [0-100]")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePercentageOfPlacementsToScore(path *field.Path, percentageOfPlacementsToScore *int32) error {
|
||||
if percentageOfPlacementsToScore != nil {
|
||||
if !feature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) {
|
||||
return field.Forbidden(path, "percentageOfPlacementsToScore requires TopologyAwareWorkloadScheduling feature gate to be enabled")
|
||||
}
|
||||
}
|
||||
return validatePercentage(path, percentageOfPlacementsToScore)
|
||||
}
|
||||
|
||||
type invalidPlugins struct {
|
||||
schemeGroupVersion string
|
||||
plugins []string
|
||||
@@ -143,7 +154,8 @@ func validateKubeSchedulerProfile(path *field.Path, apiVersion string, profile *
|
||||
if len(profile.SchedulerName) == 0 {
|
||||
errs = append(errs, field.Required(path.Child("schedulerName"), ""))
|
||||
}
|
||||
errs = append(errs, validatePercentageOfNodesToScore(path.Child("percentageOfNodesToScore"), profile.PercentageOfNodesToScore))
|
||||
errs = append(errs, validatePercentage(path.Child("percentageOfNodesToScore"), profile.PercentageOfNodesToScore))
|
||||
errs = append(errs, validatePercentageOfPlacementsToScore(path.Child("percentageOfPlacementsToScore"), profile.PercentageOfPlacementsToScore))
|
||||
errs = append(errs, validatePluginConfig(path, apiVersion, profile)...)
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -23,7 +23,10 @@ import (
|
||||
"github.com/google/go-cmp/cmp"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
componentbaseconfig "k8s.io/component-base/config"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
configv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1"
|
||||
"k8s.io/utils/ptr"
|
||||
@@ -109,6 +112,36 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) {
|
||||
percentageOfNodesToScoreNegative := validConfig.DeepCopy()
|
||||
percentageOfNodesToScoreNegative.PercentageOfNodesToScore = ptr.To[int32](-1)
|
||||
|
||||
percentageOfPlacementsToScore50 := validConfig.DeepCopy()
|
||||
percentageOfPlacementsToScore50.PercentageOfPlacementsToScore = ptr.To[int32](50)
|
||||
|
||||
percentageOfPlacementsToScore0 := validConfig.DeepCopy()
|
||||
percentageOfPlacementsToScore0.PercentageOfPlacementsToScore = ptr.To[int32](0)
|
||||
|
||||
percentageOfPlacementsToScore100 := validConfig.DeepCopy()
|
||||
percentageOfPlacementsToScore100.PercentageOfPlacementsToScore = ptr.To[int32](100)
|
||||
|
||||
percentageOfPlacementsToScore101 := validConfig.DeepCopy()
|
||||
percentageOfPlacementsToScore101.PercentageOfPlacementsToScore = ptr.To[int32](101)
|
||||
|
||||
percentageOfPlacementsToScoreNegative := validConfig.DeepCopy()
|
||||
percentageOfPlacementsToScoreNegative.PercentageOfPlacementsToScore = ptr.To[int32](-1)
|
||||
|
||||
profilePercentageOfPlacementsToScore50 := validConfig.DeepCopy()
|
||||
profilePercentageOfPlacementsToScore50.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](50)
|
||||
|
||||
profilePercentageOfPlacementsToScore0 := validConfig.DeepCopy()
|
||||
profilePercentageOfPlacementsToScore0.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](0)
|
||||
|
||||
profilePercentageOfPlacementsToScore100 := validConfig.DeepCopy()
|
||||
profilePercentageOfPlacementsToScore100.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](100)
|
||||
|
||||
profilePercentageOfPlacementsToScore101 := validConfig.DeepCopy()
|
||||
profilePercentageOfPlacementsToScore101.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](101)
|
||||
|
||||
profilePercentageOfPlacementsToScoreNegative := validConfig.DeepCopy()
|
||||
profilePercentageOfPlacementsToScoreNegative.Profiles[0].PercentageOfPlacementsToScore = ptr.To[int32](-1)
|
||||
|
||||
schedulerNameNotSet := validConfig.DeepCopy()
|
||||
schedulerNameNotSet.Profiles[1].SchedulerName = ""
|
||||
|
||||
@@ -196,8 +229,9 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) {
|
||||
invalidPlugins.Profiles[0].Plugins.Score.Enabled = append(invalidPlugins.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "GCEPDLimits"})
|
||||
|
||||
scenarios := map[string]struct {
|
||||
config *config.KubeSchedulerConfiguration
|
||||
wantErrs field.ErrorList
|
||||
featureOverrides featuregatetesting.FeatureOverrides
|
||||
config *config.KubeSchedulerConfiguration
|
||||
wantErrs field.ErrorList
|
||||
}{
|
||||
"good": {
|
||||
config: validConfig,
|
||||
@@ -256,6 +290,110 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
"percentage-of-placements-to-score-forbidden-with-tas-disabled": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.TopologyAwareWorkloadScheduling: false,
|
||||
},
|
||||
config: percentageOfPlacementsToScore50,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeForbidden,
|
||||
Field: "percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"percentage-of-placements-to-score-lower-bound": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: percentageOfPlacementsToScore0,
|
||||
},
|
||||
"percentage-of-placements-to-score-upper-bound": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: percentageOfPlacementsToScore100,
|
||||
},
|
||||
"percentage-of-placements-to-score-greater-than-100": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: percentageOfPlacementsToScore101,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"percentage-of-placements-to-score-less-than-0": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: percentageOfPlacementsToScoreNegative,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"profile-percentage-of-placements-to-score-forbidden-with-tas-disabled": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.TopologyAwareWorkloadScheduling: false,
|
||||
},
|
||||
config: profilePercentageOfPlacementsToScore50,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeForbidden,
|
||||
Field: "profiles[0].percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"profile-percentage-of-placements-to-score-lower-bound": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: profilePercentageOfPlacementsToScore0,
|
||||
},
|
||||
"profile-percentage-of-placements-to-score-upper-bound": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: profilePercentageOfPlacementsToScore100,
|
||||
},
|
||||
"profile-percentage-of-placements-to-score-greater-than-100": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: profilePercentageOfPlacementsToScore101,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "profiles[0].percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"profile-percentage-of-placements-to-score-less-than-0": {
|
||||
featureOverrides: featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
},
|
||||
config: profilePercentageOfPlacementsToScoreNegative,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "profiles[0].percentageOfPlacementsToScore",
|
||||
},
|
||||
},
|
||||
},
|
||||
"scheduler-name-not-set": {
|
||||
config: schedulerNameNotSet,
|
||||
wantErrs: field.ErrorList{
|
||||
@@ -401,6 +539,7 @@ func TestValidateKubeSchedulerConfigurationV1(t *testing.T) {
|
||||
|
||||
for name, scenario := range scenarios {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, scenario.featureOverrides)
|
||||
errs := ValidateKubeSchedulerConfiguration(scenario.config)
|
||||
diff := cmp.Diff(scenario.wantErrs.ToAggregate(), errs, ignoreBadValueDetail)
|
||||
if diff != "" {
|
||||
|
||||
+10
@@ -198,6 +198,11 @@ func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfigurati
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.PercentageOfPlacementsToScore != nil {
|
||||
in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.Profiles != nil {
|
||||
in, out := &in.Profiles, &out.Profiles
|
||||
*out = make([]KubeSchedulerProfile, len(*in))
|
||||
@@ -241,6 +246,11 @@ func (in *KubeSchedulerProfile) DeepCopyInto(out *KubeSchedulerProfile) {
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.PercentageOfPlacementsToScore != nil {
|
||||
in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.Plugins != nil {
|
||||
in, out := &in.Plugins, &out.Plugins
|
||||
*out = new(Plugins)
|
||||
|
||||
@@ -287,6 +287,9 @@ type Framework interface {
|
||||
// HasScorePlugins returns true if at least one Score plugin is defined.
|
||||
HasScorePlugins() bool
|
||||
|
||||
// HasPlacementScorePlugins returns true if at least one PlacementScore plugin is defined.
|
||||
HasPlacementScorePlugins() bool
|
||||
|
||||
// PodGroupPostFilterPlugins returns registered PodGroupPostFilter plugins.
|
||||
PodGroupPostFilterPlugins() []fwk.PodGroupPostFilterPlugin
|
||||
|
||||
@@ -296,6 +299,9 @@ type Framework interface {
|
||||
// PercentageOfNodesToScore returns percentageOfNodesToScore associated to a profile.
|
||||
PercentageOfNodesToScore() *int32
|
||||
|
||||
// PercentageOfPlacementsToScore returns percentageOfPlacementsToScore associated to a profile.
|
||||
PercentageOfPlacementsToScore() *int32
|
||||
|
||||
// SetPodNominator sets the PodNominator
|
||||
SetPodNominator(nominator fwk.PodNominator)
|
||||
// SetPodActivator sets the PodActivator
|
||||
|
||||
@@ -95,9 +95,10 @@ type frameworkImpl struct {
|
||||
|
||||
sharedCSIManager fwk.CSIManager
|
||||
|
||||
metricsRecorder *metrics.MetricAsyncRecorder
|
||||
profileName string
|
||||
percentageOfNodesToScore *int32
|
||||
metricsRecorder *metrics.MetricAsyncRecorder
|
||||
profileName string
|
||||
percentageOfNodesToScore *int32
|
||||
percentageOfPlacementsToScore *int32
|
||||
|
||||
extenders []fwk.Extender
|
||||
fwk.PodNominator
|
||||
@@ -411,6 +412,7 @@ func NewFramework(ctx context.Context, r Registry, profile *config.KubeScheduler
|
||||
|
||||
f.profileName = profile.SchedulerName
|
||||
f.percentageOfNodesToScore = profile.PercentageOfNodesToScore
|
||||
f.percentageOfPlacementsToScore = profile.PercentageOfPlacementsToScore
|
||||
if profile.Plugins == nil {
|
||||
return f, nil
|
||||
}
|
||||
@@ -427,10 +429,11 @@ func NewFramework(ctx context.Context, r Registry, profile *config.KubeScheduler
|
||||
pluginConfig[name] = profile.PluginConfig[i].Args
|
||||
}
|
||||
outputProfile := config.KubeSchedulerProfile{
|
||||
SchedulerName: f.profileName,
|
||||
PercentageOfNodesToScore: f.percentageOfNodesToScore,
|
||||
Plugins: profile.Plugins,
|
||||
PluginConfig: make([]config.PluginConfig, 0, len(pg)),
|
||||
SchedulerName: f.profileName,
|
||||
PercentageOfNodesToScore: f.percentageOfNodesToScore,
|
||||
PercentageOfPlacementsToScore: f.percentageOfPlacementsToScore,
|
||||
Plugins: profile.Plugins,
|
||||
PluginConfig: make([]config.PluginConfig, 0, len(pg)),
|
||||
}
|
||||
|
||||
f.pluginsMap = make(map[string]fwk.Plugin)
|
||||
@@ -2370,6 +2373,11 @@ func (f *frameworkImpl) HasScorePlugins() bool {
|
||||
return len(f.scorePlugins) > 0
|
||||
}
|
||||
|
||||
// HasPlacementScorePlugins returns true if at least one placementScore plugin is defined.
|
||||
func (f *frameworkImpl) HasPlacementScorePlugins() bool {
|
||||
return len(f.placementScorePlugins) > 0
|
||||
}
|
||||
|
||||
// PodGroupPostFilterPlugins returns registered PodGroupPostFilter plugins.
|
||||
func (f *frameworkImpl) PodGroupPostFilterPlugins() []fwk.PodGroupPostFilterPlugin {
|
||||
return f.podGroupPostFilterPlugins
|
||||
@@ -2470,6 +2478,11 @@ func (f *frameworkImpl) PercentageOfNodesToScore() *int32 {
|
||||
return f.percentageOfNodesToScore
|
||||
}
|
||||
|
||||
// PercentageOfPlacementsToScore returns percentageOfPlacementsToScore associated to a profile.
|
||||
func (f *frameworkImpl) PercentageOfPlacementsToScore() *int32 {
|
||||
return f.percentageOfPlacementsToScore
|
||||
}
|
||||
|
||||
// Parallelizer returns a parallelizer holding parallelism for scheduler.
|
||||
func (f *frameworkImpl) Parallelizer() fwk.Parallelizer {
|
||||
return f.parallelizer
|
||||
|
||||
@@ -72,9 +72,10 @@ const (
|
||||
placementGeneratePlugin = "placement-generate-plugin"
|
||||
defaultPreemptionPlugin = names.DefaultPreemption
|
||||
|
||||
testProfileName = "test-profile"
|
||||
testPercentageOfNodesToScore = 35
|
||||
nodeName = "testNode"
|
||||
testProfileName = "test-profile"
|
||||
testPercentageOfNodesToScore = 35
|
||||
testPercentageOfPlacementsToScore = 10
|
||||
nodeName = "testNode"
|
||||
|
||||
injectReason = "injected status"
|
||||
injectFilterReason = "injected filter status"
|
||||
@@ -4636,9 +4637,10 @@ func TestRecordingMetrics(t *testing.T) {
|
||||
|
||||
recorder := metrics.NewMetricsAsyncRecorder(100, time.Nanosecond, ctx.Done())
|
||||
profile := config.KubeSchedulerProfile{
|
||||
PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore),
|
||||
SchedulerName: testProfileName,
|
||||
Plugins: plugins,
|
||||
PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore),
|
||||
PercentageOfPlacementsToScore: ptr.To[int32](testPercentageOfPlacementsToScore),
|
||||
SchedulerName: testProfileName,
|
||||
Plugins: plugins,
|
||||
}
|
||||
f, err := newFrameworkWithQueueSortAndBind(ctx, r, profile,
|
||||
withMetricsRecorder(recorder),
|
||||
@@ -4759,9 +4761,10 @@ func TestRunBindPlugins(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
recorder := metrics.NewMetricsAsyncRecorder(100, time.Nanosecond, ctx.Done())
|
||||
profile := config.KubeSchedulerProfile{
|
||||
SchedulerName: testProfileName,
|
||||
PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore),
|
||||
Plugins: plugins,
|
||||
SchedulerName: testProfileName,
|
||||
PercentageOfNodesToScore: ptr.To[int32](testPercentageOfNodesToScore),
|
||||
PercentageOfPlacementsToScore: ptr.To[int32](testPercentageOfPlacementsToScore),
|
||||
Plugins: plugins,
|
||||
}
|
||||
fwk, err := newFrameworkWithQueueSortAndBind(ctx, r, profile, withMetricsRecorder(recorder))
|
||||
if err != nil {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"iter"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -42,6 +43,16 @@ import (
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
// minFeasiblePlacementsToFind is the minimum number of placements that would be scored
|
||||
// in each scheduling cycle.
|
||||
minFeasiblePlacementsToFind = 1
|
||||
// minFeasiblePlacementsPercentageToFind is the minimum adaptive percentage used when
|
||||
// PercentageOfPlacementsToScore is 0. An explicitly configured lower value, such as 3,
|
||||
// overrides this adaptive minimum.
|
||||
minFeasiblePlacementsPercentageToFind = 5
|
||||
)
|
||||
|
||||
// scheduleOnePodGroup does the entire workload-aware scheduling workflow for a single pod group.
|
||||
func (sched *Scheduler) scheduleOnePodGroup(ctx context.Context, podGroupInfo *framework.QueuedPodGroupInfo) {
|
||||
logger := klog.FromContext(ctx)
|
||||
@@ -927,9 +938,8 @@ func (sched *Scheduler) updateCompositePodGroupCondition(ctx context.Context,
|
||||
// First it runs placement generator plugins to create a list of placements.
|
||||
// Placement is a set of nodes that will be considered when scheduling a pod group.
|
||||
// For a standalone PodGroup it evaluates the placement matching the pods' NominatedNodeName first
|
||||
// and uses it if the gang is feasible there, short-circuiting the rest. Otherwise (or for a
|
||||
// PodGroup that is part of a CompositePodGroup) it tries every placement through
|
||||
// podGroupSchedulingDefaultAlgorithm and runs placement scorer plugins to select the best one.
|
||||
// and uses it if the gang is feasible there, short-circuiting the rest. Otherwise it evaluates
|
||||
// candidates up to the feasible-placement limit and runs placement scorer plugins to select the best one.
|
||||
func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, podGroupInfo *framework.PodGroupInfo, queuedPodGroupInfo *framework.QueuedPodGroupInfo) (finalResult *podGroupAlgorithmResult, revertFns revertFns) {
|
||||
allNodes, err := sched.nodeInfoSnapshot.ListNodesInPlacement()
|
||||
if err != nil {
|
||||
@@ -972,7 +982,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context
|
||||
// CompositePodGroup defers its feasibility verdict to the CPG root, where a Success status
|
||||
// with nothing scheduled is still meaningful. Short-circuiting on it (or dropping it) here
|
||||
// could wrongly report the whole CPG Unschedulable and stop sibling groups from being
|
||||
// evaluated, so CPG children evaluate every placement as before.
|
||||
// evaluated, so CPG children use the regular placement search.
|
||||
// TODO(kubernetes/kubernetes#140863): extend NNN support to CompositePodGroups.
|
||||
nominatedFeasible := false
|
||||
var nominated *fwk.Placement
|
||||
@@ -1001,6 +1011,8 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context
|
||||
|
||||
// Only evaluate the remaining placements when the nominated one wasn't feasible.
|
||||
if !nominatedFeasible {
|
||||
var numPlacementsToFind int
|
||||
placements, numPlacementsToFind = sched.placementsToEvaluate(schedFwk, placements)
|
||||
for _, placement := range placements {
|
||||
if placement == nominated {
|
||||
continue
|
||||
@@ -1017,6 +1029,10 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context
|
||||
if result.status.IsSuccess() {
|
||||
successfulResults[placement] = result
|
||||
}
|
||||
|
||||
if len(successfulResults) >= numPlacementsToFind {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1061,7 +1077,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context
|
||||
// compositePodGroupSchedulingPlacementAlgorithm tries several different combinations for scheduling the child pod groups and selects the best one.
|
||||
// First it runs placement generator plugins to create a list of placements.
|
||||
// Placement is a set of nodes that will be considered when scheduling a pod group.
|
||||
// Then for each placement it tries to schedule the pod group through podGroupSchedulingDefaultAlgorithm.
|
||||
// Then it evaluates candidate placements through compositePodGroupSchedulingDefaultAlgorithm up to the feasible-placement limit.
|
||||
// Finally, it runs placement scorer plugins to select the best placement.
|
||||
func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, root *framework.QueuedPodGroupInfo, podGroupInfo *framework.PodGroupInfo, results map[fwk.EntityKey]*podGroupAlgorithmResult) (finalResult *podGroupAlgorithmResult, revertFns revertFns) {
|
||||
defer func() {
|
||||
@@ -1089,6 +1105,8 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex
|
||||
var anyResultSubtree map[fwk.EntityKey]*podGroupAlgorithmResult
|
||||
successfulResults := make(map[*fwk.Placement]map[fwk.EntityKey]*podGroupAlgorithmResult)
|
||||
|
||||
placements, numPlacementsToFind := sched.placementsToEvaluate(schedFwk, placements)
|
||||
|
||||
parentPlacement := sched.nodeInfoSnapshot.GetPlacement()
|
||||
defer func() {
|
||||
sched.nodeInfoSnapshot.ForgetPlacement()
|
||||
@@ -1129,6 +1147,10 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex
|
||||
if result.status.IsSuccess() {
|
||||
successfulResults[placement] = subtreeResult
|
||||
}
|
||||
|
||||
if len(successfulResults) >= numPlacementsToFind {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(successfulResults) == 0 {
|
||||
@@ -1168,6 +1190,63 @@ func (sched *Scheduler) compositePodGroupSchedulingPlacementAlgorithm(ctx contex
|
||||
return bestResult[podGroupInfo.GetKey()], revertFns
|
||||
}
|
||||
|
||||
// randShufflePlacements is the default placement shuffler, randomizing placement order in place.
|
||||
func randShufflePlacements(placements []*fwk.Placement) {
|
||||
rand.Shuffle(len(placements), func(i, j int) {
|
||||
placements[i], placements[j] = placements[j], placements[i]
|
||||
})
|
||||
}
|
||||
|
||||
// placementsToEvaluate returns the placements in evaluation order and the number of feasible
|
||||
// placements to find before the search stops.
|
||||
func (sched *Scheduler) placementsToEvaluate(schedFwk framework.Framework, placements []*fwk.Placement) ([]*fwk.Placement, int) {
|
||||
numPlacementsToFind := 1
|
||||
if schedFwk.HasPlacementScorePlugins() {
|
||||
numPlacementsToFind = sched.numFeasiblePlacementsToFind(schedFwk.PercentageOfPlacementsToScore(), placements)
|
||||
}
|
||||
|
||||
// Only a subset of placements will be evaluated, so shuffle to avoid favoring the order
|
||||
// returned by the PlacementGenerate plugin. Clone the slice to avoid mutating plugin data.
|
||||
if sched.shufflePlacements != nil && numPlacementsToFind < len(placements) {
|
||||
placements = slices.Clone(placements)
|
||||
sched.shufflePlacements(placements)
|
||||
}
|
||||
return placements, numPlacementsToFind
|
||||
}
|
||||
|
||||
// numFeasiblePlacementsToFind returns the number of feasible placements that once found, the scheduler stops
|
||||
// its search for more feasible placements.
|
||||
func (sched *Scheduler) numFeasiblePlacementsToFind(percentageOfPlacementsToScore *int32, placements []*fwk.Placement) (numPlacements int) {
|
||||
numAllPlacements := len(placements)
|
||||
|
||||
if numAllPlacements < minFeasiblePlacementsToFind {
|
||||
return numAllPlacements
|
||||
}
|
||||
|
||||
// Use profile percentageOfPlacementsToScore if it's set. Otherwise, use global percentageOfPlacementsToScore.
|
||||
var percentage int
|
||||
if percentageOfPlacementsToScore != nil {
|
||||
percentage = int(*percentageOfPlacementsToScore)
|
||||
} else {
|
||||
percentage = int(sched.percentageOfPlacementsToScore)
|
||||
}
|
||||
|
||||
// If percentage is 0, linearly interpolate from 100% to 10% as the summed node count
|
||||
// across generated placements grows from 0 to 5000, with a 5% floor.
|
||||
if percentage == 0 {
|
||||
numAllNodes := 0
|
||||
for _, placement := range placements {
|
||||
// Placements may overlap or omit cluster nodes, so use their summed node count.
|
||||
numAllNodes += len(placement.Nodes)
|
||||
}
|
||||
percentage = max(minFeasiblePlacementsPercentageToFind, 100-numAllNodes*9/500)
|
||||
}
|
||||
|
||||
numPlacements = max(minFeasiblePlacementsToFind, numAllPlacements*percentage/100)
|
||||
|
||||
return numPlacements
|
||||
}
|
||||
|
||||
func (sched *Scheduler) findBestPodGroupPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.PodGroupInfo, successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) (*fwk.Placement, *fwk.Status) {
|
||||
if len(successfulResults) == 1 {
|
||||
for placement := range successfulResults {
|
||||
@@ -1227,8 +1306,7 @@ func (sched *Scheduler) evaluatePlacement(ctx context.Context, schedFwk framewor
|
||||
//
|
||||
// The framework contract permits overlapping placements, so a nominated node can appear in more
|
||||
// than one. NNN alone can't tell which placement WAP picked last cycle, so we only take the fast
|
||||
// path when exactly one placement matches; otherwise the caller scores every placement, which is
|
||||
// deterministic regardless of the order the generator returned them.
|
||||
// path when exactly one placement matches; otherwise the caller uses the regular placement search.
|
||||
func nominatedPlacement(placements []*fwk.Placement, podGroupInfo *framework.PodGroupInfo, queuedPodGroupInfo *framework.QueuedPodGroupInfo) *fwk.Placement {
|
||||
// Collect nominated nodes across the pods of the currently evaluated pod group node
|
||||
// (podGroupInfo), which for CPG TAS is the CPG or PG carrying the TAS constraints, not the
|
||||
@@ -1248,7 +1326,7 @@ func nominatedPlacement(placements []*fwk.Placement, podGroupInfo *framework.Pod
|
||||
for _, node := range placement.Nodes {
|
||||
if nominatedNodes.Has(node.Node().Name) {
|
||||
if matched != nil {
|
||||
// Overlap: fall back to scoring all placements.
|
||||
// Overlap: fall back to the regular placement search.
|
||||
return nil
|
||||
}
|
||||
matched = placement
|
||||
|
||||
@@ -68,6 +68,7 @@ import (
|
||||
st "k8s.io/kubernetes/pkg/scheduler/testing"
|
||||
tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
|
||||
testingclock "k8s.io/utils/clock/testing"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
// fakePodGroupPlugin simulates Filter, PostFilter, Permit and PodGroupPostFilter behaviors for PodGroup scheduling testing.
|
||||
@@ -3575,8 +3576,7 @@ func TestPodGroupSchedulingPlacementAlgorithm_NominatedNode(t *testing.T) {
|
||||
SchedulingQueue: queue,
|
||||
Profiles: profile.Map{"test-scheduler": schedFwk},
|
||||
}
|
||||
sched.initAlgorithm()
|
||||
sched.SchedulePod = sched.algorithm.SchedulePod
|
||||
initTestAlgorithm(t, sched)
|
||||
|
||||
if err := sched.Cache.UpdateSnapshot(logger, sched.nodeInfoSnapshot); err != nil {
|
||||
t.Fatalf("Failed to update snapshot: %v", err)
|
||||
@@ -8679,3 +8679,611 @@ func TestSubmitCompositePodGroupAlgorithmResult_StatusUpdates(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumFeasiblePlacementsToFind(t *testing.T) {
|
||||
makePlacements := func(placementsCount, nodesCount int) []*fwk.Placement {
|
||||
res := make([]*fwk.Placement, placementsCount)
|
||||
j := 0
|
||||
for i := range placementsCount {
|
||||
nodes := make([]fwk.NodeInfo, 0)
|
||||
for ; j < nodesCount*(i+1)/placementsCount; j++ {
|
||||
nodes = append(nodes, framework.NewNodeInfo())
|
||||
}
|
||||
res[i] = &fwk.Placement{
|
||||
Nodes: nodes,
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
globalPercentage int32
|
||||
profilePercentage *int32
|
||||
placementsCount int
|
||||
nodesCount int
|
||||
wantNumPlacements int
|
||||
}{
|
||||
{
|
||||
name: "default limit for small cluster and small number of placements",
|
||||
placementsCount: 10,
|
||||
nodesCount: 100,
|
||||
wantNumPlacements: 9,
|
||||
},
|
||||
{
|
||||
name: "default limit for large cluster and small number of placements",
|
||||
placementsCount: 20,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 2,
|
||||
},
|
||||
{
|
||||
name: "default limit for small cluster and large number of placements",
|
||||
placementsCount: 100,
|
||||
nodesCount: 100,
|
||||
wantNumPlacements: 99,
|
||||
},
|
||||
{
|
||||
name: "default limit for large cluster and large number of placements",
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 100,
|
||||
},
|
||||
{
|
||||
name: "default limit does not drop below 5%",
|
||||
placementsCount: 10000,
|
||||
nodesCount: 50000,
|
||||
wantNumPlacements: 500,
|
||||
},
|
||||
{
|
||||
name: "non-default limit can drop below 5%",
|
||||
globalPercentage: 1,
|
||||
placementsCount: 10000,
|
||||
nodesCount: 50000,
|
||||
wantNumPlacements: 100,
|
||||
},
|
||||
{
|
||||
name: "limit cannot drop below 1 placement",
|
||||
globalPercentage: 1,
|
||||
placementsCount: 10,
|
||||
nodesCount: 50000,
|
||||
wantNumPlacements: 1,
|
||||
},
|
||||
{
|
||||
name: "both global and profile limit set, profile limit lower than global",
|
||||
globalPercentage: 50,
|
||||
profilePercentage: ptr.To[int32](5),
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 50,
|
||||
},
|
||||
{
|
||||
name: "both global and profile limit set, profile limit higher than global",
|
||||
globalPercentage: 50,
|
||||
profilePercentage: ptr.To[int32](55),
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 550,
|
||||
},
|
||||
{
|
||||
name: "both global and profile limit set, profile limit using default",
|
||||
globalPercentage: 50,
|
||||
profilePercentage: ptr.To[int32](0), // use default limit
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 100,
|
||||
},
|
||||
{
|
||||
name: "only profile limit set",
|
||||
profilePercentage: ptr.To[int32](5),
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 50,
|
||||
},
|
||||
{
|
||||
name: "only global limit set",
|
||||
globalPercentage: 50,
|
||||
placementsCount: 1000,
|
||||
nodesCount: 5000,
|
||||
wantNumPlacements: 500,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sched := &Scheduler{
|
||||
percentageOfPlacementsToScore: tt.globalPercentage,
|
||||
}
|
||||
placements := makePlacements(tt.placementsCount, tt.nodesCount)
|
||||
if got := sched.numFeasiblePlacementsToFind(tt.profilePercentage, placements); got != tt.wantNumPlacements {
|
||||
t.Errorf("Scheduler.numFeasiblePlacementsToFind() = %v, want %v", got, tt.wantNumPlacements)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type trackingPlacementScorePlugin struct {
|
||||
lock sync.Mutex
|
||||
scoredPlacements sets.Set[string]
|
||||
}
|
||||
|
||||
var _ fwk.PlacementScorePlugin = &trackingPlacementScorePlugin{}
|
||||
|
||||
func (t *trackingPlacementScorePlugin) Name() string {
|
||||
return "trackingPlacementScorePlugin"
|
||||
}
|
||||
|
||||
func (t *trackingPlacementScorePlugin) PlacementScoreExtensions() fwk.PlacementScoreExtensions {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *trackingPlacementScorePlugin) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) {
|
||||
t.lock.Lock()
|
||||
defer t.lock.Unlock()
|
||||
t.scoredPlacements.Insert(placement.Name)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type trackingFilterPlugin struct {
|
||||
lock sync.Mutex
|
||||
scoredNodes sets.Set[string]
|
||||
}
|
||||
|
||||
var _ fwk.FilterPlugin = &trackingFilterPlugin{}
|
||||
|
||||
func (t *trackingFilterPlugin) Name() string {
|
||||
return "trackingFilterPlugin"
|
||||
}
|
||||
|
||||
func (t *trackingFilterPlugin) Filter(ctx context.Context, state fwk.CycleState, pod *v1.Pod, nodeInfo fwk.NodeInfo) *fwk.Status {
|
||||
t.lock.Lock()
|
||||
defer t.lock.Unlock()
|
||||
t.scoredNodes.Insert(nodeInfo.Node().Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type scheduledPodPlacementFeasiblePlugin struct{}
|
||||
|
||||
var _ fwk.PlacementFeasiblePlugin = &scheduledPodPlacementFeasiblePlugin{}
|
||||
|
||||
func (p *scheduledPodPlacementFeasiblePlugin) Name() string {
|
||||
return names.GangScheduling
|
||||
}
|
||||
|
||||
func (p *scheduledPodPlacementFeasiblePlugin) PlacementFeasible(_ context.Context, _ fwk.PlacementCycleState, _ fwk.PodGroupInfo, progress fwk.PlacementProgress) *fwk.Status {
|
||||
if progress.Remaining > 0 {
|
||||
return fwk.NewStatus(fwk.Wait)
|
||||
}
|
||||
if progress.Scheduled == 0 {
|
||||
return fwk.NewStatus(fwk.Unschedulable, "no pods scheduled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPodGroupSchedulingPlacementAlgorithm_PlacementLimit(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
features.GenericWorkload: true,
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
numFeasiblePlacements int
|
||||
numInfeasiblePlacements int
|
||||
percentageLimit int32
|
||||
skipScore bool
|
||||
expectedNumScoredPlacements int
|
||||
expectedNumCheckedPlacements *int // nil if cannot be determined due to random placement order
|
||||
}{
|
||||
{
|
||||
name: "Only limits the feasible placements",
|
||||
numFeasiblePlacements: 3,
|
||||
numInfeasiblePlacements: 97,
|
||||
percentageLimit: 2,
|
||||
expectedNumScoredPlacements: 2,
|
||||
},
|
||||
{
|
||||
name: "Checks all placements in case there is no feasible placement",
|
||||
numFeasiblePlacements: 0,
|
||||
numInfeasiblePlacements: 100,
|
||||
percentageLimit: 40,
|
||||
expectedNumScoredPlacements: 0,
|
||||
expectedNumCheckedPlacements: new(100),
|
||||
},
|
||||
{
|
||||
name: "Does not score placements if limit is 1",
|
||||
numFeasiblePlacements: 100,
|
||||
numInfeasiblePlacements: 0,
|
||||
percentageLimit: 1,
|
||||
expectedNumScoredPlacements: 0,
|
||||
expectedNumCheckedPlacements: new(1),
|
||||
},
|
||||
{
|
||||
name: "Stops after a single placement if there are no score plugins",
|
||||
numFeasiblePlacements: 100,
|
||||
numInfeasiblePlacements: 0,
|
||||
percentageLimit: 40,
|
||||
skipScore: true,
|
||||
expectedNumCheckedPlacements: new(1),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
for _, compositePodGroup := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s (CPG: %v)", test.name, compositePodGroup), func(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CompositePodGroup, compositePodGroup)
|
||||
logger, ctx := ktesting.NewTestContext(t)
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(clientsetfake.NewClientset(), 0)
|
||||
queue := internalqueue.NewSchedulingQueue(nil, informerFactory)
|
||||
|
||||
numNodes := test.numFeasiblePlacements + test.numInfeasiblePlacements
|
||||
nodes := make([]*v1.Node, numNodes)
|
||||
nodeNames := make([]string, numNodes)
|
||||
|
||||
podGroupKey := fwk.PodGroupKey("default", "pg")
|
||||
// The limit applies to the placements of the root group. A PodGroup under a
|
||||
// CompositePodGroup gets a single placement within each root placement.
|
||||
rootKey := podGroupKey
|
||||
if compositePodGroup {
|
||||
rootKey = fwk.CompositePodGroupKey("default", "cpg")
|
||||
}
|
||||
placementPlugin := &fakePlacementPlugin{
|
||||
name: "fakePlacementPlugin",
|
||||
generatePlacementsResult: map[fwk.EntityKey]map[string][]string{
|
||||
rootKey: {},
|
||||
},
|
||||
filterStatus: make(map[string]*fwk.Status),
|
||||
}
|
||||
|
||||
for i := range numNodes {
|
||||
nodeName := fmt.Sprintf("n%v", i)
|
||||
nodes[i] = st.MakeNode().Name(nodeName).UID(nodeName).Obj()
|
||||
nodeNames[i] = nodeName
|
||||
placementName := fmt.Sprintf("p%v", i)
|
||||
placementPlugin.generatePlacementsResult[rootKey][placementName] = []string{nodeName}
|
||||
placementPlugin.filterStatus[nodeName] = fwk.NewStatus(fwk.Unschedulable)
|
||||
}
|
||||
if compositePodGroup {
|
||||
placementPlugin.generatePlacementsResult[podGroupKey] = map[string][]string{"pg": nodeNames}
|
||||
}
|
||||
|
||||
feasiblePlacements := sets.New[string]()
|
||||
for placement, nodeNames := range placementPlugin.generatePlacementsResult[rootKey] {
|
||||
if len(feasiblePlacements) == test.numFeasiblePlacements {
|
||||
break
|
||||
}
|
||||
placementPlugin.filterStatus[nodeNames[0]] = fwk.NewStatus(fwk.Success)
|
||||
feasiblePlacements.Insert(placement)
|
||||
}
|
||||
|
||||
trackingFilter := &trackingFilterPlugin{scoredNodes: sets.New[string]()}
|
||||
trackingScore := &trackingPlacementScorePlugin{scoredPlacements: sets.New[string]()}
|
||||
placementFeasible := &scheduledPodPlacementFeasiblePlugin{}
|
||||
|
||||
registry := []tf.RegisterPluginFunc{
|
||||
tf.RegisterPlacementGeneratePlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return placementPlugin, nil
|
||||
}),
|
||||
tf.RegisterFilterPlugin(trackingFilter.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return trackingFilter, nil
|
||||
}),
|
||||
tf.RegisterFilterPlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return placementPlugin, nil
|
||||
}),
|
||||
tf.RegisterPlacementFeasiblePlugin(placementFeasible.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return placementFeasible, nil
|
||||
}),
|
||||
}
|
||||
|
||||
if !test.skipScore {
|
||||
registry = append(registry,
|
||||
tf.RegisterPlacementScorePlugin(trackingScore.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return trackingScore, nil
|
||||
}, 1),
|
||||
)
|
||||
}
|
||||
|
||||
cache := internalcache.New(ctx, nil, true, compositePodGroup)
|
||||
for _, node := range nodes {
|
||||
cache.AddNode(logger, node)
|
||||
}
|
||||
testPodGroup := st.MakePodGroup().Name("pg").Namespace("default").Obj()
|
||||
testCompositePodGroup := st.MakeCompositePodGroup().Name("cpg").Namespace("default").Obj()
|
||||
cache.AddGenericPodGroup(fwk.NewGenericPodGroup(testPodGroup))
|
||||
if compositePodGroup {
|
||||
cache.AddGenericPodGroup(fwk.NewGenericCompositePodGroup(testCompositePodGroup))
|
||||
}
|
||||
snapshot := internalcache.NewEmptySnapshot()
|
||||
if err := cache.UpdateSnapshot(logger, snapshot); err != nil {
|
||||
t.Fatalf("Failed to update snapshot: %v", err)
|
||||
}
|
||||
|
||||
schedFwk, err := tf.NewFramework(ctx,
|
||||
append(registry,
|
||||
tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
|
||||
tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
|
||||
),
|
||||
"test-scheduler",
|
||||
frameworkruntime.WithInformerFactory(informerFactory),
|
||||
frameworkruntime.WithSnapshotSharedLister(snapshot),
|
||||
frameworkruntime.WithPodNominator(queue),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create new framework: %v", err)
|
||||
}
|
||||
|
||||
sched := &Scheduler{
|
||||
Cache: cache,
|
||||
nodeInfoSnapshot: snapshot,
|
||||
SchedulingQueue: queue,
|
||||
Profiles: profile.Map{"test-scheduler": schedFwk},
|
||||
percentageOfPlacementsToScore: test.percentageLimit,
|
||||
}
|
||||
initTestAlgorithm(t, sched)
|
||||
|
||||
podGroupPod := st.MakePod().Name("foo").Namespace("default").UID("foo").PodGroupName("pg").Obj()
|
||||
runPlacementAlgorithm(ctx, t, sched, schedFwk, testPodGroup, testCompositePodGroup, compositePodGroup, podGroupPod)
|
||||
|
||||
if got := len(trackingScore.scoredPlacements); got != test.expectedNumScoredPlacements {
|
||||
t.Errorf("Unexpected number of scored placements, want %d, got %d", test.expectedNumScoredPlacements, got)
|
||||
}
|
||||
|
||||
if len(trackingScore.scoredPlacements) > 0 {
|
||||
for skippedPlacement := range feasiblePlacements.Difference(trackingScore.scoredPlacements) {
|
||||
placementNode := placementPlugin.generatePlacementsResult[rootKey][skippedPlacement][0]
|
||||
if trackingFilter.scoredNodes.Has(placementNode) {
|
||||
t.Errorf("Unexpected checked node for skipped placement %s, want none, got %v", skippedPlacement, placementNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if test.expectedNumCheckedPlacements != nil {
|
||||
// 1 unique node per placement
|
||||
if got := len(trackingFilter.scoredNodes); got != *test.expectedNumCheckedPlacements {
|
||||
t.Errorf("Unexpected number of checked placements, want %d, got %d", *test.expectedNumCheckedPlacements, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runPlacementAlgorithm runs the placement algorithm for podGroup, or for a CompositePodGroup with
|
||||
// podGroup as its only child when compositePodGroup is true, and reverts the assumed result.
|
||||
func runPlacementAlgorithm(ctx context.Context, t *testing.T, sched *Scheduler, schedFwk framework.Framework, podGroup *schedulingv1beta1.PodGroup, cpg *schedulingv1alpha3.CompositePodGroup, compositePodGroup bool, pod *v1.Pod) *podGroupAlgorithmResult {
|
||||
t.Helper()
|
||||
podGroupInfo := &framework.PodGroupInfo{GenericPodGroup: fwk.NewGenericPodGroup(podGroup)}
|
||||
podInfo := &framework.QueuedPodInfo{PodInfo: &framework.PodInfo{Pod: pod}}
|
||||
var result *podGroupAlgorithmResult
|
||||
var revert revertFns
|
||||
if compositePodGroup {
|
||||
cpgInfo := newQueuedPodGroupInfo(&framework.PodGroupInfo{
|
||||
GenericPodGroup: fwk.NewGenericCompositePodGroup(cpg),
|
||||
Children: []*framework.PodGroupInfo{podGroupInfo},
|
||||
}, podInfo)
|
||||
result, revert = sched.compositePodGroupSchedulingPlacementAlgorithm(ctx, schedFwk, framework.NewCycleState(), cpgInfo, cpgInfo.PodGroupInfo, map[fwk.EntityKey]*podGroupAlgorithmResult{})
|
||||
} else {
|
||||
pgInfo := newQueuedPodGroupInfo(podGroupInfo, podInfo)
|
||||
result, revert = sched.podGroupSchedulingPlacementAlgorithm(ctx, schedFwk, framework.NewCycleState(), pgInfo.PodGroupInfo, pgInfo)
|
||||
}
|
||||
revert.revert()
|
||||
return result
|
||||
}
|
||||
|
||||
// fixedOrderPlacementPlugin returns one placement per node of the parent placement in a
|
||||
// deterministic order, so a test can verify that the scheduler itself randomizes the evaluation
|
||||
// order. It records the first slice it returns to verify that the scheduler doesn't mutate it.
|
||||
type fixedOrderPlacementPlugin struct {
|
||||
placementNames []string
|
||||
nodePerPlacement map[string]string
|
||||
firstPlacements []*fwk.Placement
|
||||
}
|
||||
|
||||
func (p *fixedOrderPlacementPlugin) Name() string { return "fixedOrderPlacementPlugin" }
|
||||
|
||||
func (p *fixedOrderPlacementPlugin) GeneratePlacements(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, parentPlacement *fwk.Placement) (*fwk.GeneratePlacementsResult, *fwk.Status) {
|
||||
parentNodes := map[string]fwk.NodeInfo{}
|
||||
for _, node := range parentPlacement.Nodes {
|
||||
parentNodes[node.Node().Name] = node
|
||||
}
|
||||
placements := make([]*fwk.Placement, 0, len(p.placementNames))
|
||||
for _, name := range p.placementNames {
|
||||
if node, ok := parentNodes[p.nodePerPlacement[name]]; ok {
|
||||
placements = append(placements, &fwk.Placement{
|
||||
Name: name,
|
||||
Nodes: []fwk.NodeInfo{node},
|
||||
})
|
||||
}
|
||||
}
|
||||
if p.firstPlacements == nil {
|
||||
p.firstPlacements = placements
|
||||
}
|
||||
return &fwk.GeneratePlacementsResult{Placements: placements}, nil
|
||||
}
|
||||
|
||||
func TestPodGroupSchedulingPlacementAlgorithm_UsesShufflePlacements(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
features.GenericWorkload: true,
|
||||
})
|
||||
|
||||
const numPlacements = 10
|
||||
// The test shuffler moves this placement to the front.
|
||||
const shuffledFirstPlacement = "p7"
|
||||
allNodes := sets.New[string]()
|
||||
for i := range numPlacements {
|
||||
allNodes.Insert(fmt.Sprintf("n%v", i))
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
compositePodGroup bool
|
||||
percentageLimit int32
|
||||
nominatedNodeName string
|
||||
wantCheckedNodes sets.Set[string]
|
||||
wantShuffled bool
|
||||
}{
|
||||
{
|
||||
name: "first shuffled placement",
|
||||
percentageLimit: 10,
|
||||
wantCheckedNodes: sets.New("n7"),
|
||||
wantShuffled: true,
|
||||
},
|
||||
{
|
||||
name: "nominated placement is used without shuffling",
|
||||
percentageLimit: 10,
|
||||
nominatedNodeName: "n0",
|
||||
wantCheckedNodes: sets.New("n0"),
|
||||
},
|
||||
{
|
||||
name: "no shuffling when every placement may be scored",
|
||||
percentageLimit: 100,
|
||||
wantCheckedNodes: allNodes,
|
||||
},
|
||||
{
|
||||
name: "first shuffled placement of a CompositePodGroup",
|
||||
compositePodGroup: true,
|
||||
percentageLimit: 10,
|
||||
wantCheckedNodes: sets.New("n7"),
|
||||
wantShuffled: true,
|
||||
},
|
||||
{
|
||||
name: "no shuffling of CompositePodGroup placements when every placement may be scored",
|
||||
compositePodGroup: true,
|
||||
percentageLimit: 100,
|
||||
wantCheckedNodes: allNodes,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CompositePodGroup, test.compositePodGroup)
|
||||
logger, ctx := ktesting.NewTestContext(t)
|
||||
|
||||
placementPlugin := &fixedOrderPlacementPlugin{nodePerPlacement: map[string]string{}}
|
||||
nodes := make([]*v1.Node, numPlacements)
|
||||
for i := range numPlacements {
|
||||
nodeName := fmt.Sprintf("n%v", i)
|
||||
placementName := fmt.Sprintf("p%v", i)
|
||||
nodes[i] = st.MakeNode().Name(nodeName).UID(nodeName).Obj()
|
||||
placementPlugin.placementNames = append(placementPlugin.placementNames, placementName)
|
||||
placementPlugin.nodePerPlacement[placementName] = nodeName
|
||||
}
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(clientsetfake.NewClientset(), 0)
|
||||
queue := internalqueue.NewSchedulingQueue(nil, informerFactory)
|
||||
|
||||
cache := internalcache.New(ctx, nil, true, test.compositePodGroup)
|
||||
for _, node := range nodes {
|
||||
cache.AddNode(logger, node)
|
||||
}
|
||||
testPodGroup := st.MakePodGroup().Name("pg").Namespace("default").Obj()
|
||||
testCompositePodGroup := st.MakeCompositePodGroup().Name("cpg").Namespace("default").Obj()
|
||||
cache.AddGenericPodGroup(fwk.NewGenericPodGroup(testPodGroup))
|
||||
if test.compositePodGroup {
|
||||
cache.AddGenericPodGroup(fwk.NewGenericCompositePodGroup(testCompositePodGroup))
|
||||
}
|
||||
snapshot := internalcache.NewEmptySnapshot()
|
||||
if err := cache.UpdateSnapshot(logger, snapshot); err != nil {
|
||||
t.Fatalf("Failed to update snapshot: %v", err)
|
||||
}
|
||||
|
||||
trackingFilter := &trackingFilterPlugin{scoredNodes: sets.New[string]()}
|
||||
trackingScore := &trackingPlacementScorePlugin{scoredPlacements: sets.New[string]()}
|
||||
registry := []tf.RegisterPluginFunc{
|
||||
tf.RegisterPlacementGeneratePlugin(placementPlugin.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return placementPlugin, nil
|
||||
}),
|
||||
tf.RegisterFilterPlugin(trackingFilter.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return trackingFilter, nil
|
||||
}),
|
||||
tf.RegisterPlacementScorePlugin(trackingScore.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) {
|
||||
return trackingScore, nil
|
||||
}, 1),
|
||||
tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
|
||||
tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
|
||||
}
|
||||
|
||||
schedFwk, err := tf.NewFramework(ctx, registry, "test-scheduler",
|
||||
frameworkruntime.WithInformerFactory(informerFactory),
|
||||
frameworkruntime.WithSnapshotSharedLister(snapshot),
|
||||
frameworkruntime.WithPodNominator(queue),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create new framework: %v", err)
|
||||
}
|
||||
|
||||
shuffled := false
|
||||
sched := &Scheduler{
|
||||
Cache: cache,
|
||||
nodeInfoSnapshot: snapshot,
|
||||
SchedulingQueue: queue,
|
||||
Profiles: profile.Map{"test-scheduler": schedFwk},
|
||||
percentageOfPlacementsToScore: test.percentageLimit,
|
||||
shufflePlacements: func(placements []*fwk.Placement) {
|
||||
shuffled = true
|
||||
for i, p := range placements {
|
||||
if p.Name == shuffledFirstPlacement {
|
||||
placements[0], placements[i] = placements[i], placements[0]
|
||||
return
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
initTestAlgorithm(t, sched)
|
||||
|
||||
podGroupPod := st.MakePod().Name("foo").Namespace("default").UID("foo").PodGroupName("pg").NominatedNodeName(test.nominatedNodeName).Obj()
|
||||
result := runPlacementAlgorithm(ctx, t, sched, schedFwk, testPodGroup, testCompositePodGroup, test.compositePodGroup, podGroupPod)
|
||||
if !result.status.IsSuccess() {
|
||||
t.Fatalf("Expected successful placement, got %v", result.status)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(sets.List(test.wantCheckedNodes), sets.List(trackingFilter.scoredNodes)); diff != "" {
|
||||
t.Errorf("Unexpected checked nodes (-want,+got):\n%s", diff)
|
||||
}
|
||||
if shuffled != test.wantShuffled {
|
||||
t.Errorf("Unexpected shuffle call, want %v, got %v", test.wantShuffled, shuffled)
|
||||
}
|
||||
var gotPluginOrder []string
|
||||
for _, p := range placementPlugin.firstPlacements {
|
||||
gotPluginOrder = append(gotPluginOrder, p.Name)
|
||||
}
|
||||
if diff := cmp.Diff(placementPlugin.placementNames, gotPluginOrder); diff != "" {
|
||||
t.Errorf("Placements returned by the plugin were mutated (-want,+got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewShufflePlacementsRandomizesOrder(t *testing.T) {
|
||||
client := clientsetfake.NewClientset()
|
||||
informerFactory := informers.NewSharedInformerFactory(client, 0)
|
||||
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
|
||||
_, ctx := ktesting.NewTestContext(t)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
sched, err := New(ctx, client, informerFactory, nil, profile.NewRecorderFactory(eventBroadcaster))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create scheduler: %v", err)
|
||||
}
|
||||
if sched.shufflePlacements == nil {
|
||||
t.Fatal("Expected New to set a non-nil shufflePlacements function")
|
||||
}
|
||||
|
||||
const numPlacements = 10
|
||||
const runs = 50
|
||||
firstNames := sets.New[string]()
|
||||
for range runs {
|
||||
placements := make([]*fwk.Placement, numPlacements)
|
||||
for i := range placements {
|
||||
placements[i] = &fwk.Placement{Name: fmt.Sprintf("p%d", i)}
|
||||
}
|
||||
sched.shufflePlacements(placements)
|
||||
firstNames.Insert(placements[0].Name)
|
||||
}
|
||||
|
||||
if len(firstNames) <= 1 {
|
||||
t.Errorf("Expected the default shuffler to randomize placement order, but saw only %d distinct first placement(s) over %d runs", len(firstNames), runs)
|
||||
}
|
||||
}
|
||||
@@ -98,6 +98,12 @@ type Scheduler struct {
|
||||
|
||||
nodeInfoSnapshot *internalcache.Snapshot
|
||||
|
||||
percentageOfPlacementsToScore int32
|
||||
|
||||
// shufflePlacements randomizes candidate placement order so the placement limit picks a fair
|
||||
// subset. Set via New; nil disables shuffling to keep tests deterministic.
|
||||
shufflePlacements func(placements []*fwk.Placement)
|
||||
|
||||
// logger *must* be initialized when creating a Scheduler,
|
||||
// otherwise logging functions will access a nil sink and
|
||||
// panic.
|
||||
@@ -127,6 +133,7 @@ type schedulerOptions struct {
|
||||
kubeConfig *restclient.Config
|
||||
// Overridden by profile level percentageOfNodesToScore if set in v1.
|
||||
percentageOfNodesToScore int32
|
||||
percentageOfPlacementsToScore int32
|
||||
podInitialBackoffSeconds int64
|
||||
podMaxBackoffSeconds int64
|
||||
podMaxInUnschedulablePodsDuration time.Duration
|
||||
@@ -199,6 +206,16 @@ func WithPercentageOfNodesToScore(percentageOfNodesToScore *int32) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithPercentageOfPlacementsToScore sets percentageOfPlacementsToScore for Scheduler.
|
||||
// The default value of 0 will use an adaptive percentage.
|
||||
func WithPercentageOfPlacementsToScore(percentageOfPlacementsToScore *int32) Option {
|
||||
return func(o *schedulerOptions) {
|
||||
if percentageOfPlacementsToScore != nil {
|
||||
o.percentageOfPlacementsToScore = *percentageOfPlacementsToScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithFrameworkOutOfTreeRegistry sets the registry for out-of-tree plugins. Those plugins
|
||||
// will be appended to the default registry.
|
||||
func WithFrameworkOutOfTreeRegistry(registry frameworkruntime.Registry) Option {
|
||||
@@ -262,6 +279,7 @@ func WithBuildFrameworkCapturer(fc FrameworkCapturer) Option {
|
||||
var defaultSchedulerOptions = schedulerOptions{
|
||||
clock: clock.RealClock{},
|
||||
percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
|
||||
percentageOfPlacementsToScore: schedulerapi.DefaultPercentageOfPlacementsToScore,
|
||||
podInitialBackoffSeconds: int64(internalqueue.DefaultPodInitialBackoffDuration.Seconds()),
|
||||
podMaxBackoffSeconds: int64(internalqueue.DefaultPodMaxBackoffDuration.Seconds()),
|
||||
podMaxInUnschedulablePodsDuration: internalqueue.DefaultPodMaxInUnschedulablePodsDuration,
|
||||
@@ -374,6 +392,8 @@ func New(ctx context.Context,
|
||||
Cache: schedulerCache,
|
||||
client: client,
|
||||
nodeInfoSnapshot: snapshot,
|
||||
percentageOfPlacementsToScore: options.percentageOfPlacementsToScore,
|
||||
shufflePlacements: randShufflePlacements,
|
||||
StopEverything: stopEverything,
|
||||
SchedulingQueue: podQueue,
|
||||
Profiles: profiles,
|
||||
|
||||
@@ -470,6 +470,51 @@ func TestWithPercentageOfNodesToScore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithPercentageOfPlacementsToScore tests scheduler's PercentageOfPlacementsToScore is set correctly.
|
||||
func TestWithPercentageOfPlacementsToScore(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
percentageOfPlacementsToScoreConfig *int32
|
||||
wantedPercentageOfPlacementsToScore int32
|
||||
}{
|
||||
{
|
||||
name: "percentageOfPlacementsScore is nil",
|
||||
percentageOfPlacementsToScoreConfig: nil,
|
||||
wantedPercentageOfPlacementsToScore: schedulerapi.DefaultPercentageOfPlacementsToScore,
|
||||
},
|
||||
{
|
||||
name: "percentageOfPlacementsScore is not nil",
|
||||
percentageOfPlacementsToScoreConfig: ptr.To[int32](10),
|
||||
wantedPercentageOfPlacementsToScore: 10,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := fake.NewClientset()
|
||||
informerFactory := informers.NewSharedInformerFactory(client, 0)
|
||||
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
|
||||
_, ctx := ktesting.NewTestContext(t)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
sched, err := New(
|
||||
ctx,
|
||||
client,
|
||||
informerFactory,
|
||||
nil,
|
||||
profile.NewRecorderFactory(eventBroadcaster),
|
||||
WithPercentageOfPlacementsToScore(tt.percentageOfPlacementsToScoreConfig),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create scheduler: %v", err)
|
||||
}
|
||||
if sched.percentageOfPlacementsToScore != tt.wantedPercentageOfPlacementsToScore {
|
||||
t.Errorf("scheduler.percentageOfPlacementsToScore = %v, want %v", sched.percentageOfPlacementsToScore, tt.wantedPercentageOfPlacementsToScore)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// getPodFromPriorityQueue is the function used in the TestDefaultErrorFunc test to get
|
||||
// the specific pod from the given priority queue. It returns the found pod in the priority queue.
|
||||
func getPodFromPriorityQueue(queue *internalqueue.PriorityQueue, pod *v1.Pod) *v1.Pod {
|
||||
|
||||
@@ -68,6 +68,23 @@ type KubeSchedulerConfiguration struct {
|
||||
// nodes will be scored. It is overridden by profile level PercentageOfNodesToScore.
|
||||
PercentageOfNodesToScore *int32 `json:"percentageOfNodesToScore,omitempty"`
|
||||
|
||||
// PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler
|
||||
// tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and
|
||||
// scores them. Valid values are from 0 to 100.
|
||||
// The scheduler always tries to find at least one feasible placement.
|
||||
// Example: if 500 placements are generated and the value is 30, the scheduler stops after
|
||||
// finding 150 feasible placements.
|
||||
// When unset or 0, the scheduler calculates an adaptive percentage based on the sum of node
|
||||
// counts across all generated placements. If placements overlap, a node present in multiple
|
||||
// placements is counted once per placement. The adaptive percentage decreases linearly from
|
||||
// 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger
|
||||
// sums down to a minimum of 5%.
|
||||
// It is overridden by profile level PercentageOfPlacementsToScore.
|
||||
// This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.
|
||||
// +featureGate=TopologyAwareWorkloadScheduling
|
||||
// +optional
|
||||
PercentageOfPlacementsToScore *int32 `json:"percentageOfPlacementsToScore,omitempty"`
|
||||
|
||||
// PodInitialBackoffSeconds is the initial backoff for unschedulable pods.
|
||||
// If specified, it must be greater than 0. If this value is null, the default value (1s)
|
||||
// will be used.
|
||||
@@ -153,6 +170,24 @@ type KubeSchedulerProfile struct {
|
||||
// global PercentageOfNodesToScore will be used.
|
||||
PercentageOfNodesToScore *int32 `json:"percentageOfNodesToScore,omitempty"`
|
||||
|
||||
// PercentageOfPlacementsToScore is the percentage of generated placements that the scheduler
|
||||
// tries to find feasible for a PodGroup or CompositePodGroup before it stops searching and
|
||||
// scores them. Valid values are from 0 to 100.
|
||||
// The scheduler always tries to find at least one feasible placement.
|
||||
// Example: if 500 placements are generated and the value is 30, the scheduler stops after
|
||||
// finding 150 feasible placements.
|
||||
// When the value is 0, the scheduler calculates an adaptive percentage based on the sum of node
|
||||
// counts across all generated placements. If placements overlap, a node present in multiple
|
||||
// placements is counted once per placement. The adaptive percentage decreases linearly from
|
||||
// 100% at 0 summed nodes to 10% at 5000 summed nodes, and continues decreasing for larger
|
||||
// sums down to a minimum of 5%.
|
||||
// It overrides global PercentageOfPlacementsToScore. If empty, the global
|
||||
// PercentageOfPlacementsToScore is used.
|
||||
// This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.
|
||||
// +featureGate=TopologyAwareWorkloadScheduling
|
||||
// +optional
|
||||
PercentageOfPlacementsToScore *int32 `json:"percentageOfPlacementsToScore,omitempty"`
|
||||
|
||||
// Plugins specify the set of plugins that should be enabled or disabled.
|
||||
// Enabled plugins are the ones that should be enabled in addition to the
|
||||
// default plugins. Disabled plugins are any of the default plugins that
|
||||
|
||||
@@ -218,6 +218,11 @@ func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfigurati
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.PercentageOfPlacementsToScore != nil {
|
||||
in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.PodInitialBackoffSeconds != nil {
|
||||
in, out := &in.PodInitialBackoffSeconds, &out.PodInitialBackoffSeconds
|
||||
*out = new(int64)
|
||||
@@ -276,6 +281,11 @@ func (in *KubeSchedulerProfile) DeepCopyInto(out *KubeSchedulerProfile) {
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.PercentageOfPlacementsToScore != nil {
|
||||
in, out := &in.PercentageOfPlacementsToScore, &out.PercentageOfPlacementsToScore
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
if in.Plugins != nil {
|
||||
in, out := &in.Plugins, &out.Plugins
|
||||
*out = new(Plugins)
|
||||
|
||||
@@ -807,6 +807,8 @@ type PlacementGeneratePlugin interface {
|
||||
|
||||
// GeneratePlacements generates a list of potential Placements for the given PodGroup within the parent placement.
|
||||
// Each Placement represents a candidate set of resources, e.g., nodes matching a selector.
|
||||
// The scheduler may shuffle and evaluate only a subset of the returned Placements, so plugins
|
||||
// must not rely on their order.
|
||||
GeneratePlacements(ctx context.Context, state PodGroupCycleState, podGroup PodGroupInfo, parentPlacement *Placement) (*GeneratePlacementsResult, *Status)
|
||||
}
|
||||
|
||||
|
||||
@@ -2008,16 +2008,200 @@ func TestCPGTopologyAwareSchedulingWorkloadAwarePreemption(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func runCPGTestScenario(t *testing.T, tt scenario) {
|
||||
func TestCPGTopologyAwareSchedulingWithPlacementLimit(t *testing.T) {
|
||||
tests := []scenario{
|
||||
{
|
||||
name: "parent CPG co-locates children on a single rack despite a restrictive placement limit",
|
||||
steps: []stepsframework.Step{
|
||||
{
|
||||
Name: "Create nodes in multiple racks, each rack able to fit the whole hierarchy",
|
||||
CreateNodes: []*v1.Node{
|
||||
makeNode("node1-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node2-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node3-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node4-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node5-z2-r3", "rack-3", "zone-2"),
|
||||
makeNode("node6-z2-r3", "rack-3", "zone-2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=rack)",
|
||||
CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "rack", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg1 (Gang with minCount=2, without topology constraints, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg2 (Gang with minCount=2, without topology constraints, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU",
|
||||
CreatePods: []*v1.Pod{
|
||||
makePod("p1", "pg1"),
|
||||
makePod("p2", "pg1"),
|
||||
makePod("p3", "pg2"),
|
||||
makePod("p4", "pg2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods in the composite group are scheduled",
|
||||
WaitForPodsScheduled: []string{"p1", "p2", "p3", "p4"},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods across both children landed on a single rack despite the limit",
|
||||
VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{
|
||||
Pods: []string{"p1", "p2", "p3", "p4"},
|
||||
TopologyKey: "rack",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restrictive placement limit still finds the only feasible rack for a CPG",
|
||||
steps: []stepsframework.Step{
|
||||
{
|
||||
Name: "Create nodes in multiple racks; only rack-1 can fit the whole hierarchy",
|
||||
CreateNodes: []*v1.Node{
|
||||
makeNode("node1-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node2-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node3-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node4-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node5-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node6-z2-r3", "rack-3", "zone-2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Block rack-2 so it can no longer fit the whole hierarchy",
|
||||
CreatePods: []*v1.Pod{
|
||||
makeAssignedPod("existing1", "node4-z1-r2", "2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=rack)",
|
||||
CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "rack", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg1 (Gang with minCount=2, without topology constraints, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg2 (Gang with minCount=1, without topology constraints, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "", 1),
|
||||
},
|
||||
{
|
||||
Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU",
|
||||
CreatePods: []*v1.Pod{
|
||||
makePod("p1", "pg1"),
|
||||
makePod("p2", "pg1"),
|
||||
makePod("p3", "pg2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods in the composite group are scheduled",
|
||||
WaitForPodsScheduled: []string{"p1", "p2", "p3"},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods landed on the only feasible rack",
|
||||
VerifyAssignments: &stepsframework.VerifyAssignments{
|
||||
Pods: []string{"p1", "p2", "p3"},
|
||||
Nodes: sets.New("node1-z1-r1", "node2-z1-r1", "node3-z1-r1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restrictive placement limit applies at each level of a CPG hierarchy (root=zone, leaf=rack)",
|
||||
steps: []stepsframework.Step{
|
||||
{
|
||||
Name: "Create nodes across zones and racks",
|
||||
CreateNodes: []*v1.Node{
|
||||
makeNode("node1-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node2-z1-r1", "rack-1", "zone-1"),
|
||||
makeNode("node3-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node4-z1-r2", "rack-2", "zone-1"),
|
||||
makeNode("node5-z2-r3", "rack-3", "zone-2"),
|
||||
makeNode("node6-z2-r3", "rack-3", "zone-2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Block zone-2 so it can no longer fit the whole hierarchy",
|
||||
CreatePods: []*v1.Pod{
|
||||
makeAssignedPod("existing1", "node6-z2-r3", "2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Create the root CompositePodGroup object (Gang with minGroupCount=2, TopologyKey=zone)",
|
||||
CreateCompositePodGroup: makeGangCompositePodGroup("cpg-root", "", "zone", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg1 (Gang with minCount=2, TopologyKey=rack, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg1", "cpg-root", "rack", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create child PodGroup pg2 (Gang with minCount=2, TopologyKey=rack, Parent=cpg-root)",
|
||||
CreatePodGroup: makeGangPodGroupWithParent("pg2", "cpg-root", "rack", 2),
|
||||
},
|
||||
{
|
||||
Name: "Create all pods belonging to pg1 and pg2, each pod requiring 1 CPU",
|
||||
CreatePods: []*v1.Pod{
|
||||
makePod("p1", "pg1"),
|
||||
makePod("p2", "pg1"),
|
||||
makePod("p3", "pg2"),
|
||||
makePod("p4", "pg2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods in the composite group are scheduled",
|
||||
WaitForPodsScheduled: []string{"p1", "p2", "p3", "p4"},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods landed on the only feasible zone",
|
||||
VerifyAssignments: &stepsframework.VerifyAssignments{
|
||||
Pods: []string{"p1", "p2", "p3", "p4"},
|
||||
Nodes: sets.New("node1-z1-r1", "node2-z1-r1", "node3-z1-r2", "node4-z1-r2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify pg1 landed on a single rack",
|
||||
VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{
|
||||
Pods: []string{"p1", "p2"},
|
||||
TopologyKey: "rack",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify pg2 landed on a single rack",
|
||||
VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{
|
||||
Pods: []string{"p3", "p4"},
|
||||
TopologyKey: "rack",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// A low percentage forces the scheduler to score only a subset of the generated placements.
|
||||
percentageLimit := int32(1)
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
runCPGTestScenario(t, tt, scheduler.WithPercentageOfPlacementsToScore(new(percentageLimit)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runCPGTestScenario(t *testing.T, tt scenario, opts ...scheduler.Option) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{
|
||||
features.CompositePodGroup: true,
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
})
|
||||
|
||||
testCtx := testutils.InitTestSchedulerWithNS(t, "cpg-tas",
|
||||
opts = append([]scheduler.Option{
|
||||
scheduler.WithPodMaxBackoffSeconds(0),
|
||||
scheduler.WithPodInitialBackoffSeconds(0))
|
||||
scheduler.WithPodInitialBackoffSeconds(0),
|
||||
}, opts...)
|
||||
testCtx := testutils.InitTestSchedulerWithNS(t, "cpg-tas", opts...)
|
||||
ns := testCtx.NS.Name
|
||||
|
||||
if err := stepsframework.RunSteps(testCtx, t, ns, tt.steps); err != nil {
|
||||
|
||||
@@ -1752,3 +1752,125 @@ func runTestScenario(t *testing.T, tt scenario, cpgEnabled bool) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopologyAwareSchedulingWithPlacementLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
steps []stepsframework.Step
|
||||
}{
|
||||
{
|
||||
name: "basic podgroup co-locates on a single rack despite a restrictive placement limit",
|
||||
steps: []stepsframework.Step{
|
||||
{
|
||||
Name: "Create nodes in multiple racks, each rack able to fit the whole group",
|
||||
CreateNodes: []*v1.Node{
|
||||
makeNode("node1-rack1", "rack-1", "zone-1"),
|
||||
makeNode("node2-rack1", "rack-1", "zone-1"),
|
||||
makeNode("node3-rack2", "rack-2", "zone-1"),
|
||||
makeNode("node4-rack2", "rack-2", "zone-1"),
|
||||
makeNode("node5-rack3", "rack-3", "zone-2"),
|
||||
makeNode("node6-rack3", "rack-3", "zone-2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Create the PodGroup constrained to a single rack",
|
||||
CreatePodGroup: makeBasicPodGroup("pg1", "rack"),
|
||||
},
|
||||
{
|
||||
Name: "Create all pods belonging to the podgroup",
|
||||
CreatePods: []*v1.Pod{
|
||||
makePod("p1", "pg1"),
|
||||
makePod("p2", "pg1"),
|
||||
makePod("p3", "pg1"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify the entire podgroup is scheduled",
|
||||
WaitForPodsScheduled: []string{"p1", "p2", "p3"},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods landed on a single rack despite the limit",
|
||||
VerifyAssignedInOneDomain: &stepsframework.VerifyAssignedInOneDomain{
|
||||
Pods: []string{"p1", "p2", "p3"},
|
||||
TopologyKey: "rack",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restrictive placement limit still finds the only feasible rack for a gang",
|
||||
steps: []stepsframework.Step{
|
||||
{
|
||||
Name: "Create nodes in multiple racks; only rack-1 can fit the whole gang",
|
||||
CreateNodes: []*v1.Node{
|
||||
makeNode("node1-rack1", "rack-1", "zone-1"),
|
||||
makeNode("node2-rack1", "rack-1", "zone-1"),
|
||||
makeNode("node3-rack1", "rack-1", "zone-1"),
|
||||
makeNode("node4-rack2", "rack-2", "zone-1"),
|
||||
makeNode("node5-rack2", "rack-2", "zone-1"),
|
||||
makeNode("node6-zone2", "rack-3", "zone-2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Block rack-2 so it can no longer fit the whole gang",
|
||||
CreatePods: []*v1.Pod{
|
||||
makeAssignedPod("existing1", "node4-rack2", "2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Create the PodGroup object (Gang with minCount=3) constrained to a single rack",
|
||||
CreatePodGroup: makeGangPodGroup("pg1", "rack", 3),
|
||||
},
|
||||
{
|
||||
Name: "Create all pods belonging to the gang",
|
||||
CreatePods: []*v1.Pod{
|
||||
makePod("p1", "pg1"),
|
||||
makePod("p2", "pg1"),
|
||||
makePod("p3", "pg1"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Verify the entire gang is scheduled",
|
||||
WaitForPodsScheduled: []string{"p1", "p2", "p3"},
|
||||
},
|
||||
{
|
||||
Name: "Verify all pods landed on the only feasible rack",
|
||||
VerifyAssignments: &stepsframework.VerifyAssignments{
|
||||
Pods: []string{"p1", "p2", "p3"},
|
||||
Nodes: sets.New("node1-rack1", "node2-rack1", "node3-rack1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// A low percentage forces the scheduler to score only a subset of the generated placements.
|
||||
percentageLimit := int32(1)
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{
|
||||
features.GenericWorkload: true,
|
||||
features.TopologyAwareWorkloadScheduling: true,
|
||||
})
|
||||
|
||||
testCtx := testutils.InitTestSchedulerWithNS(t, "tas-placement-limit",
|
||||
scheduler.WithPodMaxBackoffSeconds(0),
|
||||
scheduler.WithPodInitialBackoffSeconds(0),
|
||||
scheduler.WithPercentageOfPlacementsToScore(new(percentageLimit)))
|
||||
ns := testCtx.NS.Name
|
||||
|
||||
workload := st.MakeWorkload().Name("workload").Namespace(ns).
|
||||
PodGroupTemplate(st.MakePodGroupTemplate().Name("t1").MinCount(1).Obj()).
|
||||
Obj()
|
||||
workloadStep := []stepsframework.Step{
|
||||
{
|
||||
Name: "Creating workload",
|
||||
CreateWorkloads: []*schedulingapi.Workload{workload},
|
||||
},
|
||||
}
|
||||
if err := stepsframework.RunSteps(testCtx, t, ns, append(workloadStep, tt.steps...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,7 @@ func StartSchedulerWithDone(tCtx ktesting.TContext, cfg *kubeschedulerconfig.Kub
|
||||
scheduler.WithKubeConfig(tCtx.RESTConfig()),
|
||||
scheduler.WithProfiles(cfg.Profiles...),
|
||||
scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore),
|
||||
scheduler.WithPercentageOfPlacementsToScore(cfg.PercentageOfPlacementsToScore),
|
||||
scheduler.WithPodMaxBackoffSeconds(cfg.PodMaxBackoffSeconds),
|
||||
scheduler.WithPodInitialBackoffSeconds(cfg.PodInitialBackoffSeconds),
|
||||
scheduler.WithExtenders(cfg.Extenders...),
|
||||
|
||||
Reference in new issue
Block a user