mirrors/linux
0
0 Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-10-26 10:57:38 +08:00
Code Activity

Merge tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Browse Source 
Pull sched_ext updates from Tejun Heo:

 - Proxy execution can now be enabled together with sched_ext. A BPF
   scheduler can opt in to receive mutex-blocked tasks that donate their
   scheduling context to the lock owner and place them itself.

 - Lazy preemption: a BPF scheduler can preempt or kick a CPU lazily,
   deferring the switch to the next return to user space or tick, as the
   fair class can.

 - Sub-schedulers: a parent now learns which child runs on each CPU it
   delegated and when a cap change takes effect. A detaching child loses
   its caps before the parent restores the CPUs.

 - New kfuncs re-enqueue single tasks from a dispatch queue iteration
   and report a cgroup's CPU count and a CID's NUMA node. The CID
   topology now includes clusters.

 - Hot path optimizations: dispatch queue and task lookups compare keys
   inline, remote task moves no longer take a dispatch queue lock twice,
   and sub-scheduler hooks cost nothing without sub-schedulers.

 - Kfuncs callable from tracing programs are hardened against NMI and
   IRQ nesting, which could hard-lock the CPU or pick a wrong idle CPU.

 - Built-in dispatch queues no longer allocate per-CPU state that grew
   with the square of the CPU count.

 - Header syncs with the scx repo, a BTF workaround for pahole 1.32, and
   selftest fixes and coverage.

* tag 'sched_ext-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (69 commits)
  sched_ext: Drop the locks periodically while walking a user DSQ for re-enqueue
  sched_ext: scx_qmap: Also re-enqueue stranded tasks with scx_bpf_dsq_reenq_task()
  sched_ext: Add scx_bpf_dsq_reenq_task() to re-enqueue single tasks from a DSQ iteration
  sched_ext: Factor out the rq lock switch of scx_dsq_move()
  sched_ext: Don't queue balance callbacks from a core-sched pick
  sched_ext: Don't leave balance callbacks queued across a pick retry
  sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on
  sched_ext: Fix the running task keeping its protection into bypass
  sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu
  sched_ext: Keep proxy donors with slice left on the local DSQ
  selftests/sched_ext: Report reload_loop attach failures
  selftests/sched_ext: Bound exit waits
  selftests/sched_ext: Add SCX_ASSERT_ALIVE and adopt it
  selftests/sched_ext: Fail loudly when EXIT_KIND is missing from BTF
  sched_ext: Add SCX_ECODE_RSN_CGROUP_OFFLINE to user_exit_info.h
  selftests/sched_ext: Fix reload_loop error-path use-after-free
  selftests/sched_ext: Fix build error in dequeue_remote selftest
  sched_ext: scx_qmap: Reset a child's cpuperf targets once its PERF revoke takes effect
  sched_ext: Add ops.sub_child_ecaps_updated() to report a child's effective cap changes
  sched_ext: scx_qmap: Show actual cid use per participant
  ...
This commit is contained in:
Linus Torvalds committed 2026-10-22 10:37:35 -10:00
2 parents 77f77fb792 1f5dde349a
commit 98aec86f8c
65 files changed
+6166 -719
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/scheduler/sched-ext.rst
+13
View File
@@ -503,6 +503,19 @@ and edge cases, to name a few examples:
class, in which case it will exit the tick-dispatch loop even though it is runnable
and has a non-zero slice.
* Under proxy execution, sched_ext continues to observe the donor as the
current scheduling context. Accordingly, ``ops.running()`` and
``ops.stopping()`` report when the donor's scheduling context becomes active
and inactive, even when the donor is blocked and a lock owner executes on its
behalf. The physical execution context is intentionally not reported through
these callbacks.
A blocked donor enters a running session only after proxy resolution finds
an execution context. The session remains active if only the physical
execution context changes while the donor remains the same. Running sessions
are tracked so that ``ops.running()`` and ``ops.stopping()`` remain paired
and are not emitted recursively.
See the "Scheduling Cycle" section for a more detailed description of how
a freshly woken up task gets on a CPU.
include/linux/sched/ext.h
+26 -13
View File
@@ -91,7 +91,7 @@ struct scx_dispatch_q {
struct rhash_head hash_node;
struct llist_node free_node;
struct scx_sched *sched;
struct scx_dsq_pcpu __percpu *pcpu;
struct scx_dsq_pcpu __percpu *pcpu_user;
struct rcu_head rcu;
};
@@ -104,6 +104,7 @@ enum scx_ent_flags {
SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */
SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */
SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */
/*
* Bits 8 to 10 are used to carry task state:
@@ -136,6 +137,7 @@ enum scx_ent_flags {
* IMMED reenqueued due to failed ENQ_IMMED
* PREEMPTED preempted while running
* CAP sub-sched cap miss, see p->scx.reenq_reason_*
* PROXY proxy state prevented a remote DSQ transfer
*/
SCX_TASK_REENQ_REASON_SHIFT = 12,
SCX_TASK_REENQ_REASON_BITS = 3,
@@ -146,6 +148,7 @@ enum scx_ent_flags {
SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT,
SCX_TASK_REENQ_PROXY = 5 << SCX_TASK_REENQ_REASON_SHIFT,
/* iteration cursor, not a task */
SCX_TASK_CURSOR = 1 << 31,
@@ -154,6 +157,7 @@ enum scx_ent_flags {
/* scx_entity.dsq_flags */
enum scx_ent_dsq_flags {
SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */
SCX_TASK_DSQ_REENQ = 1 << 1, /* marked by scx_bpf_dsq_reenq_task() */
};
enum scx_dsq_lnode_flags {
@@ -197,9 +201,19 @@ struct sched_ext_entity {
u64 ddsp_slice;
u64 ddsp_vtime;
struct scx_dsq_list_node dsq_list; /* dispatch order */
struct rb_node dsq_priq; /* p->scx.dsq_vtime order */
u32 dsq_seq;
u32 dsq_flags; /* protected by DSQ lock */
/*
* Used to order tasks when dispatching to the vtime-ordered priority
* queue of a dsq. This is usually set through
* scx_bpf_dsq_insert_vtime() but can also be modified directly by the
* BPF scheduler. Modifying it while a task is queued on a dsq may
* mangle the ordering and is not recommended. Kept next to @dsq_priq
* as rbtree insertion reads both on every visited node.
*/
u64 dsq_vtime;
struct rb_node dsq_priq; /* p->scx.dsq_vtime order */
u32 flags; /* protected by rq lock */
u32 weight;
u32 reenq_cnt; /* reenqueues since last run */
@@ -226,7 +240,7 @@ struct sched_ext_entity {
u64 tid;
struct rhash_head tid_hash_node; /* see SCX_OPS_TID_TO_TASK */
/* BPF scheduler modifiable fields */
/* BPF scheduler modifiable fields, along with @dsq_vtime above */
/*
* Runtime budget in nsecs - how long the task may hold its cpu. Owned
@@ -241,15 +255,6 @@ struct sched_ext_entity {
*/
u64 slice;
/*
* Used to order tasks when dispatching to the vtime-ordered priority
* queue of a dsq. This is usually set through
* scx_bpf_dsq_insert_vtime() but can also be modified directly by the
* BPF scheduler. Modifying it while a task is queued on a dsq may
* mangle the ordering and is not recommended.
*/
u64 dsq_vtime;
/*
* Out-of-band slice request from scx_bpf_task_set_slice() when the
* caller does not hold the rq lock, applied under the rq lock at the
@@ -279,6 +284,14 @@ struct sched_ext_entity {
*/
bool disallow; /* reject switching into SCX */
/*
* If set, depletion of this task's slice at the scheduler tick requests
* lazy instead of immediate rescheduling. Initialized from
* %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be
* modified afterwards with scx_bpf_task_set_lazy_resched().
*/
bool lazy_resched;
/* cold fields */
#ifdef CONFIG_EXT_GROUP_SCHED
struct cgroup *cgrp_moving_from;
@@ -324,7 +337,7 @@ struct scx_task_group {
u64 bw_period_us;
u64 bw_quota_us;
u64 bw_burst_us;
bool idle;
bool sched_idle;
#endif
};
init/Kconfig
-2
View File
@@ -743,8 +743,6 @@ config SCHED_PROXY_EXEC
bool "Proxy Execution"
# Avoid some build failures w/ PREEMPT_RT until it can be fixed
depends on !PREEMPT_RT
# Need to investigate how to inform sched_ext of split contexts
depends on !SCHED_CLASS_EXT
# Not particularly useful until we get to multi-rq proxying
depends on EXPERT
help
kernel/sched/ext/cid.c
+58 -9
View File
@@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */
#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \
.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \
.node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \
.cluster_cid = -1, .cluster_idx = -1, \
}
/*
@@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch)
cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
struct scx_cid_tables *tbls;
u32 next_cid = 0;
s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
s32 next_cluster_idx = 0;
s32 next_shard_idx = 0;
u32 shard_size, max_cids;
u32 notopo_in_shard;
@@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch)
if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
!zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&core_scratch, GFP_KERNEL) ||
!zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) ||
!zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
@@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch)
u32 cores_per_shard, nr_large;
u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
s32 shard_cid, shard_idx;
s32 cluster_cid = -1, cluster_idx = -1;
/* llc_scratch = node_scratch & this llc */
cpumask_and(llc_scratch, node_scratch, llc_mask);
@@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch)
tbls->shard_node[shard_idx] = nid;
while (!cpumask_empty(llc_scratch)) {
s32 lcpu = cpumask_first(llc_scratch);
const struct cpumask *sib = topology_sibling_cpumask(lcpu);
s32 core_cid = next_cid;
s32 core_idx = next_core_idx++;
s32 ccpu;
const struct cpumask *sib;
s32 core_cid, core_idx, lcpu, ccpu;
u32 max_cores, cids_in_core;
/* core_scratch = llc_scratch & this core */
cpumask_and(core_scratch, llc_scratch, sib);
/*
* Take the cores of one cluster before moving
* on, so that a cluster is a contiguous cid
* range like the core, LLC and node levels.
*/
if (cpumask_empty(cluster_scratch)) {
s32 xcpu = cpumask_first(llc_scratch);
cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu),
topology_sibling_cpumask(xcpu));
cpumask_and(cluster_scratch, cluster_scratch, llc_scratch);
cluster_cid = next_cid;
cluster_idx = next_cluster_idx++;
}
lcpu = cpumask_first(cluster_scratch);
sib = topology_sibling_cpumask(lcpu);
core_cid = next_cid;
core_idx = next_core_idx++;
/* core_scratch = cluster_scratch & this core */
cpumask_and(core_scratch, cluster_scratch, sib);
if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
return -EINVAL;
@@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch)
.node_idx = node_idx,
.shard_cid = shard_cid,
.shard_idx = shard_idx,
.cluster_cid = cluster_cid,
.cluster_idx = cluster_idx,
};
cpumask_clear_cpu(ccpu, cluster_scratch);
cpumask_clear_cpu(ccpu, llc_scratch);
cpumask_clear_cpu(ccpu, node_scratch);
cpumask_clear_cpu(ccpu, to_walk);
@@ -461,8 +486,8 @@ __bpf_kfunc_start_defs();
* starts must be strictly increasing with the first entry 0 and all values <
* num_possible_cpus(). The last shard extends to num_possible_cpus() and no
* shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
* (core/LLC/node) is cleared and the shard layout is set from the input. On
* invalid input, abort the scheduler.
* (core/cluster/LLC/node) is cleared and the shard layout is set from the
* input. On invalid input, abort the scheduler.
*/
__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt,
const s32 *shard_start__arena, u32 shard_start_cnt,
@@ -640,6 +665,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux)
return scx_cid_to_cpu(sch, cid);
}
/**
* scx_bpf_cid_node - Return the NUMA node the given @cid belongs to
* @cid: cid to look up
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike
* scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF
* schedulers can use its presence to detect NUMA-aware CID lookups.
*/
__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux)
{
struct scx_sched *sch;
s32 cpu;
guard(rcu)();
sch = scx_prog_sched(aux);
if (unlikely(!sch))
return NUMA_NO_NODE;
cpu = scx_cid_to_cpu(sch, cid);
return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu);
}
/**
* scx_bpf_cpu_to_cid - Return the cid for @cpu
* @cpu: cpu to look up
@@ -958,6 +1006,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
BTF_KFUNCS_START(scx_kfunc_ids_cid)
BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS)
BTF_KFUNCS_END(scx_kfunc_ids_cid)
kernel/sched/ext/cid.h
+6 -5
View File
@@ -13,11 +13,12 @@
* kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences
* for every op.
*
* cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or
* NUMA node get contiguous cid ranges, so a topology unit becomes a (start,
* length) slice of cid space. Communication can pass a slice instead of a
* cpumask, and BPF code can process, for example, a u64 word's worth of cids at
* a time.
* cids give every cpu a dense, topology-ordered id. CPUs in each core,
* cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit
* becomes a (start, length) slice of cid space. A core without a wider cluster
* level forms a cluster of its own. Communication can pass a slice
* instead of a cpumask, and BPF code can process, for example, a u64 word's
* worth of cids at a time.
*
* The mapping is built once at root scheduler enable time by walking the
* topology of online cpus only. Going by online cpus is out of necessity:
kernel/sched/ext/ext.c
+1019 -250
View File
File diff suppressed because it is too large. Load diff
kernel/sched/ext/ext.h
+19 -13
View File
@@ -18,12 +18,28 @@ bool scx_can_stop_tick(struct rq *rq);
void scx_rq_activate(struct rq *rq);
void scx_rq_deactivate(struct rq *rq);
int scx_check_setscheduler(struct task_struct *p, int policy);
void scx_prepare_setscheduler(struct task_struct *p, int policy);
bool task_should_scx(int policy);
bool scx_allow_ttwu_queue(const struct task_struct *p);
bool scx_allow_proxy_exec(const struct task_struct *p);
bool __scx_allow_proxy_exec(const struct task_struct *p);
void scx_proxy_donor_start(struct rq *rq);
void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next);
void init_sched_ext_class(void);
void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
static inline bool scx_allow_proxy_exec(const struct task_struct *p)
{
if (scx_enabled())
return __scx_allow_proxy_exec(p);
return true;
}
static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
{
if (scx_enabled())
__scx_update_idle(rq, idle, do_notify);
}
static inline u32 scx_cpuperf_target(s32 cpu)
{
@@ -55,6 +71,7 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; }
static inline void scx_rq_activate(struct rq *rq) {}
static inline void scx_rq_deactivate(struct rq *rq) {}
static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; }
static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {}
static inline bool task_on_scx(const struct task_struct *p) { return false; }
static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; }
static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; }
@@ -62,21 +79,10 @@ static inline void scx_proxy_donor_start(struct rq *rq) {}
static inline void scx_proxy_reenqueue_retry(struct rq *rq,
struct task_struct *next) {}
static inline void init_sched_ext_class(void) {}
static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
#endif /* CONFIG_SCHED_CLASS_EXT */
#ifdef CONFIG_SCHED_CLASS_EXT
void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
{
if (scx_enabled())
__scx_update_idle(rq, idle, do_notify);
}
#else
static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
#endif
#ifdef CONFIG_CGROUP_SCHED
#ifdef CONFIG_EXT_GROUP_SCHED
void scx_tg_init(struct task_group *tg);
kernel/sched/ext/idle.c
+12 -2
View File
@@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
nodemask_t *unvisited;
s32 cpu = -EBUSY;
preempt_disable();
/*
* @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be
* called from IRQ-enabled contexts, so mask IRQs to keep a nested
* invocation from clobbering the mask an outer invocation is still
* iterating.
*
* NMI nesting is not handled: there is no legitimate reason to call
* pick_idle from NMI and doing so poses no crash risk, so such a
* caller is on its own.
*/
guard(irqsave)();
unvisited = this_cpu_ptr(&per_cpu_unvisited);
/*
@@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
if (cpu >= 0)
break;
}
preempt_enable();
return cpu;
}
kernel/sched/ext/internal.h
+189 -7
View File
@@ -215,6 +215,34 @@ enum scx_ops_flags {
*/
SCX_OPS_TID_TO_TASK = 1LLU << 8,
/*
* If set, tasks default to requesting lazy rescheduling when their
* slice runs out at the tick, the way fair.c expires a slice from
* update_curr(). The default is copied to p->scx.lazy_resched
* immediately before ops.enable(), after which
* scx_bpf_task_set_lazy_resched() may override it per task.
*
* A task in user space still reschedules on the way back from the tick;
* a task in the kernel runs on to its next return to user space or to
* the next tick, which promotes the request. When lazy preemption is
* disabled at runtime, the request behaves like an immediate
* reschedule. Rescheduling while bypassing stays immediate.
*/
SCX_OPS_LAZY_RESCHED = 1LLU << 9,
/*
* If set, mutex-blocked tasks remain runnable as proxy donors and are
* passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls
* when donors are dispatched and whether they should preempt other work.
*
* If clear, mutex-blocked tasks are removed from the runqueue normally
* and cannot donate their scheduling context through proxy execution.
*
* For blocked donors, this flag takes precedence over
* %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED.
*/
SCX_OPS_ENQ_BLOCKED = 1LLU << 10,
SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
SCX_OPS_ENQ_LAST |
SCX_OPS_ENQ_EXITING |
@@ -223,7 +251,9 @@ enum scx_ops_flags {
SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_BUILTIN_IDLE_PER_NODE |
SCX_OPS_ALWAYS_ENQ_IMMED |
SCX_OPS_TID_TO_TASK,
SCX_OPS_TID_TO_TASK |
SCX_OPS_LAZY_RESCHED |
SCX_OPS_ENQ_BLOCKED,
/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
__SCX_OPS_INTERNAL_MASK = 0xffLLU << 56,
@@ -336,6 +366,18 @@ struct scx_sub_detach_args {
char *cgroup_path;
};
/*
* What a sched sees running on a cid, as reported by
* ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside
* the sched's subtree, SELF for the sched itself, or a direct child by its
* cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits
* are an idr value of at most INT_MAX.
*/
enum scx_cid_sched_consts {
SCX_CID_SCHED_NONE = (u64)-2,
SCX_CID_SCHED_SELF = (u64)-1,
};
/**
* struct sched_ext_ops - Operation table for BPF scheduler implementation
*
@@ -477,6 +519,12 @@ struct sched_ext_ops {
* Therefore, always use scx_bpf_task_cpu(@p) to determine the
* target CPU the task is going to use.
*
* Under proxy execution, the BPF scheduler continues to observe the
* donor as the current scheduling context. A blocked donor enters a
* ->running()/->stopping() session while its scheduling context drives
* the lock owner. The lock owner executing on its behalf is intentionally
* not reported through these callbacks.
*
* See ->runnable() for explanation on the task state notifiers.
*/
void (*running)(struct task_struct *p);
@@ -828,6 +876,11 @@ struct sched_ext_ops {
/**
* @sub_detach: Detach a sub-scheduler
* @args: argument container, see the struct definition
*
* The sub-scheduler holds no caps by this point and can no longer
* affect any cid. Its caps went away without a
* sub_child_ecaps_updated() report, so whatever was delegated to it,
* e.g. cpuperf targets, is the parent's to restore here.
*/
void (*sub_detach)(struct scx_sub_detach_args *args);
@@ -864,6 +917,41 @@ struct sched_ext_ops {
*/
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
/**
* @sub_cid_sched_updated: The sched running on a cid changed
* @cid: cid whose running sched changed
* @sched: what this sched now sees running there
*
* Invoked when the sched owning the task running on @cid changes.
* @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree
* runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of
* the direct child whose subtree the task belongs to. Changes inside a
* child's subtree, whether across tasks or nested sub-scheds, are not
* reported. Every sched for which @sched changes is notified.
*
* Runs with the rq lock held on the context switch path.
*/
void (*sub_cid_sched_updated)(s32 cid, u64 sched);
/**
* @sub_child_ecaps_updated: A child's effective caps on a cid changed
* @cgroup_id: cgroup id of the direct child
* @cid: the cid whose effective caps changed
* @before: the child's effective caps as of the last delivery
* @after: the child's effective caps now
*
* Invoked right after the child's sub_ecaps_updated(), once a grant or
* revoke is in effect on the cpu. From here on the parent can act on a
* cap the child lost, e.g. reset the cpuperf target or schedule on the
* cid. A detaching child's caps go away without a report, see
* sub_detach(). A cpu going offline drops caps without a report.
*
* Runs in dispatch context with rq lock held, and can perform all
* operations allowed in ops.dispatch() including inserting/moving
* tasks.
*/
void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after);
/*
* All online ops must come before ops.cpu_online().
*/
@@ -911,6 +999,9 @@ struct sched_ext_ops {
* ops.exit() is also called on ops.init() failure, which is a bit
* unusual. This is to allow rich reporting through @info on how
* ops.init() failed.
*
* A sub-scheduler holds no caps by the time ops.exit() runs. Its
* cap-gated kfuncs are denied.
*/
void (*exit)(struct scx_exit_info *info);
@@ -1118,6 +1209,8 @@ struct sched_ext_ops_cid {
void (*sub_detach)(struct scx_sub_detach_args *args);
void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps);
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
void (*sub_cid_sched_updated)(s32 cid, u64 sched);
void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after);
void (*cid_online)(s32 cid);
void (*cid_offline)(s32 cid);
s32 (*init_cids)(void);
@@ -1359,6 +1452,7 @@ struct scx_sched_pcpu {
cpumask_var_t cpus_to_kick;
cpumask_var_t cpus_to_kick_if_idle;
cpumask_var_t cpus_to_preempt;
cpumask_var_t cpus_to_preempt_lazy;
cpumask_var_t cpus_to_wait;
struct list_head to_kick_node;
@@ -1380,7 +1474,10 @@ struct scx_sched_pcpu {
struct llist_node ecaps_to_sync_node;
/* owed a forced update_idle() re-notify on this cpu */
bool idle_renotify;
/* effective caps as of the last sub_ecaps_updated() delivery */
/*
* ecaps as of the last sub_ecaps_updated() and
* sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps().
*/
u64 reported_ecaps;
/*
@@ -1439,15 +1536,16 @@ struct scx_sched_pnode {
* the allocation pattern.
*
* ENQ_IMMED insert an IMMED task onto the cid's local DSQ
* - kick the cid's cpu (except SCX_KICK_PREEMPT)
* - kick the cid's cpu (except SCX_KICK_PREEMPT and
* SCX_KICK_PREEMPT_LAZY)
*
* ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED)
*
* PREEMPT preempt any task running on the cid regardless of the owning
* sched (implies ENQ). Preempting a task in the sched's own subtree
* doesn't require any cap.
* - SCX_ENQ_PREEMPT inserts
* - SCX_KICK_PREEMPT kicks
* - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert
* - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick
*
* PERF control the cid's cpu power/perf management state, currently the
* cpufreq target set through scx_bpf_cidperf_set(). Hardware
@@ -1732,11 +1830,20 @@ enum scx_enq_flags {
*/
SCX_ENQ_PREEMPT = 1LLU << 32,
/*
* Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current
* task's slice is still cleared immediately so that the next scheduling
* boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence
* if both are specified. Implies %SCX_ENQ_HEAD, which is all it means
* on a non-local DSQ, as with %SCX_ENQ_PREEMPT.
*/
SCX_ENQ_PREEMPT_LAZY = 1LLU << 35,
/*
* Only allowed on local DSQs. Guarantees that the task either gets
* on the CPU immediately and stays on it, or gets reenqueued back
* to the BPF scheduler. It will never linger on a local DSQ or be
* silently put back after preemption.
* to the BPF scheduler. A blocked proxy donor may be put back on the
* local DSQ temporarily while proxy execution resolves its mutex owner.
*
* The protection persists until the next fresh enqueue - it
* survives SAVE/RESTORE cycles, slice extensions and preemption.
@@ -1780,6 +1887,18 @@ enum scx_enq_flags {
*/
SCX_ENQ_LAST = 1LLU << 41,
/*
* The task is blocked on a mutex and is being kept runnable as a proxy
* donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set.
*
* Blocking on the mutex does not enqueue the task by itself. A donor put
* with slice left stays at the head of the local DSQ, except when an
* IMMED donor is preempted or cannot stay local. It is passed to
* ops.enqueue() when its slice runs out, the IMMED placement cannot be
* kept, or proxy execution moves it to the owner's CPU.
*/
SCX_ENQ_BLOCKED = 1LLU << 42,
/* high 8 bits are internal */
__SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56,
@@ -1792,6 +1911,15 @@ enum scx_enq_flags {
SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */
};
/* Strip priority and carried slice state when diverting from a local DSQ. */
static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags)
{
*enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT |
SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD |
SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
p->scx.flags &= ~SCX_TASK_IMMED;
}
enum scx_deq_flags {
/* expose select DEQUEUE_* flags as enums */
SCX_DEQ_SLEEP = DEQUEUE_SLEEP,
@@ -1818,6 +1946,8 @@ enum scx_reenq_flags {
/* internal: kernel-issued on cap revoke, not accepted from BPF */
SCX_REENQ_CAP_REVOKE = 1LLU << 1,
/* internal: tasks marked by scx_bpf_dsq_reenq_task() */
SCX_REENQ_MARKED = 1LLU << 2,
__SCX_REENQ_FILTER_MASK = 0xffffLLU,
@@ -1859,6 +1989,15 @@ enum scx_kick_flags {
* is not on SCX.
*/
SCX_KICK_WAIT = 1LLU << 2,
/*
* Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined
* with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate.
*/
SCX_KICK_PREEMPT_LAZY = 1LLU << 3,
SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT |
SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY,
};
enum scx_tg_flags {
@@ -2107,6 +2246,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
bool scx_set_task_slice(struct task_struct *p, u64 slice);
void scx_task_slice_ended(struct rq *rq, struct task_struct *p);
void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
void scx_prepare_task_sched_change(struct task_struct *p);
void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
int sticky_cpu);
@@ -2168,6 +2308,22 @@ extern struct scx_sched *scx_enabling_sub_sched;
#define scx_error(sch, fmt, args...) \
scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
/*
* Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or
* touch the kick lists, which are only protected by irq masking, can't run
* there, so abort the scheduler instead. scx_error() is NMI-safe.
*/
static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who)
{
if (unlikely(in_nmi())) {
scx_error(sch, "%s called from NMI", who);
return false;
}
return true;
}
#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__)
/**
* scx_root_protected_live - Root sched for paths that only run while live
*
@@ -2258,6 +2414,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags)
*/
static inline struct rq *scx_locked_rq(void)
{
/*
* Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks
* the rq locked by the interrupted context, so a non-NULL read from
* NMI would falsely claim its lock. Return NULL from NMI so that
* callers take their unlocked paths.
*/
if (unlikely(in_nmi()))
return NULL;
return __this_cpu_read(scx_locked_rq_state);
}
@@ -2539,4 +2704,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL;
#endif /* CONFIG_EXT_SUB_SCHED */
/**
* scx_exec_sched - Find the sched running @sch's tasks on @cpu
* @sch: sched owning the tasks
* @cpu: cpu of interest
*
* A bypassing sched's tasks are run by its nearest ancestor that isn't
* bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that
* ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for
* a stable answer.
*/
static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu)
{
while (scx_parent(sch) && scx_bypassing(sch, cpu))
sch = scx_parent(sch);
return sch;
}
#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */
kernel/sched/ext/sub.c
+290 -97
View File
@@ -28,6 +28,13 @@
*/
DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
/*
* On while a loaded sched implements ops.sub_cid_sched_updated(), see
* scx_ops_cid_sched_updated_enable().
*/
static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled);
static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */
/* latched at root enable before any rescue runs */
static s32 scx_rescue_bw_1024;
static s64 scx_rescue_quantum_ns;
@@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche
return scx_skip_subtree_pre(pos, root);
}
/**
* scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid
* @sch: sched whose task runs there, %NULL for none
* @from: sched asking
*
* Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree,
* %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct
* child of @from on the way down to @sch.
*/
static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from)
{
if (!sch || !scx_is_descendant(sch, from))
return SCX_CID_SCHED_NONE;
if (sch == from)
return SCX_CID_SCHED_SELF;
return sch->ancestors[from->level + 1]->ops.sub_cgroup_id;
}
/* notify @from of what it now sees running on @rq's cid */
static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch,
struct scx_sched *from)
{
if (SCX_HAS_OP(from, sub_cid_sched_updated))
SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)),
scx_cid_sched_id_from(sch, from));
}
/**
* scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected
* @rq: rq whose running session opens or closes
* @sch: sched starting a session, %NULL to close the open one
*
* Every sched whose scx_cid_sched_id_from() value changes is notified of the
* new one. Above the deepest common ancestor of the two scheds, both map to the
* same child, so the walk stops there. With root R, children A and B, and A's
* child A1, a switch from an A1 task to a B task reports:
*
* R R: A -> B
* / \
* A B A: A1 -> NONE B: NONE -> SELF
* |
* A1 A1: SELF -> NONE
*
* When the task on one side is not an ext task, there is no common ancestor and
* only the other side is walked.
*/
void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch)
{
struct scx_sched *prev = rq->scx.sched;
s32 level, common = -1;
lockdep_assert_rq_held(rq);
if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch)
return;
rq->scx.sched = sch;
if (prev && sch) {
for (level = min(prev->level, sch->level); level >= 0; level--)
if (prev->ancestors[level] == sch->ancestors[level])
break;
common = level;
scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]);
}
if (prev)
for (level = common + 1; level <= prev->level; level++)
scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]);
if (sch)
for (level = common + 1; level <= sch->level; level++)
scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]);
}
/**
* scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op
* @sch: sched being enabled, its has_op filled in
*
* rq->scx.sched is maintained only while a loaded sched implements
* ops.sub_cid_sched_updated(), so that everyone else pays one static branch per
* session open or close. The first such sched turns the key on and then
* initializes every rq from the task running there, in that order so that no
* session change in between goes unrecorded. While the key is off every
* rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable().
*/
void scx_ops_cid_sched_updated_enable(struct scx_sched *sch)
{
s32 cpu;
lockdep_assert_held(&scx_enable_mutex);
if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++)
return;
static_branch_enable(&__scx_ops_cid_sched_updated_enabled);
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
struct task_struct *p;
guard(rq_lock_irqsave)(rq);
p = rq->donor;
if (p->sched_class == &ext_sched_class &&
(p->scx.flags & SCX_TASK_RUN_TRACKED))
rq->scx.sched = scx_task_sched(p);
}
}
/**
* scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch
* @sch: sched being disabled, its tasks all gone
*
* Close the session on every rq that still points at @sch, before ops.exit(),
* so that no sched is notified of @sch once it is gone. The last sched with the
* op turns the key off first and clears every rq, so while the key is off every
* rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was
* never counted, so has_op gates the decrement.
*/
void scx_ops_cid_sched_updated_disable(struct scx_sched *sch)
{
bool last;
s32 cpu;
lockdep_assert_held(&scx_enable_mutex);
last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated;
if (last)
static_branch_disable(&__scx_ops_cid_sched_updated_enabled);
/*
* Re-homing restarts a running task's session under its new sched. A
* task that blocked while its cpu's dispatch had the rq lock dropped is
* put and set next again without a restart, so rq->scx.sched keeps @sch
* until the pick completes:
*
* cpu 0, __schedule() cpu 1, scx_sub_disable(@sch)
* T of @sch blocks, stops running
* dispatch drops the rq lock
* re-home T, nothing to restart
* this sweep
* pick N, N's session opens
*
* Closing the session here reports NONE to @sch and its ancestors now
* and lets the pick report N's sched from NONE instead of from @sch.
*/
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
guard(rq_lock_irqsave)(rq);
if (rq->scx.sched == sch)
scx_cid_sched_update(rq, NULL);
if (last)
rq->scx.sched = NULL;
}
}
static struct scx_sched *scx_find_sub_sched(u64 cgroup_id)
{
return rhashtable_lookup(&scx_sched_hash, &cgroup_id,
@@ -285,7 +447,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec)
rq->scx.rescue.budget -= delta_exec;
/* per-cpu usage average feeds the overload victim pick */
pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu,
cpu_of(rq));
pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec;
if (!scx_rescue_slice_remaining(rq))
@@ -557,7 +720,7 @@ static void scx_rescue_timerfn(struct timer_list *timer)
scx_rescue_admit(rq, p, slice);
scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
SCX_ENQ_IGNORE_CAPS, rq);
if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
if (sched_class_above(&ext_sched_class, rq->donor->sched_class))
resched_curr(rq);
} else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
/*
@@ -671,7 +834,7 @@ void scx_rescue_init(struct rq *rq)
}
/**
* scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
* __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
* @sch: enqueuing sub-sched
* @rq: rq whose local DSQ @p targets
* @p: task being inserted
@@ -682,20 +845,18 @@ void scx_rescue_init(struct rq *rq)
* rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
* @p.
*
* %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting
* to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback
* migration-disabled admission.
* %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are
* cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and
* %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled
* admission.
*
* Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued
* to and run by its nearest non-bypassing ancestor. If root is bypassing, it
* always holds all caps.
*/
struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags)
struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags)
{
if (!scx_has_subs())
return &rq->scx.local_dsq;
s32 cid = __scx_cpu_to_cid(cpu_of(rq));
struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
u64 needed = scx_caps_for_enq(*enq_flags);
@@ -705,7 +866,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
* On a remote activation the scheduling sched (@asch) differs from
* @p's owner (@sch). Check caps against the scheduling sched.
*/
if (*enq_flags & SCX_ENQ_PREEMPT)
if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
missing = scx_missing_caps(asch, cpu_of(rq), needed);
@@ -722,7 +883,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
p->migration_pending)) {
__scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1);
*enq_flags &= ~SCX_ENQ_PREEMPT;
*enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY);
return &rq->scx.local_dsq;
}
@@ -731,9 +892,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
* or HEAD - a diversion has no priority and IMMED is not allowed on
* non-local DSQs. Strip the enq and task flags along with the slice.
*/
*enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD |
SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
p->scx.flags &= ~SCX_TASK_IMMED;
scx_divert_strip_flags(p, enq_flags);
/* the enqueuer opted for rescue instead of rejection and reenqueue */
if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) {
@@ -748,6 +907,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
p->scx.reenq_reason_caps = missing;
p->scx.reenq_reason_cid = cid;
WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
p->scx.flags |= SCX_TASK_REENQ_CAP;
return &rq->scx.reject_dsq;
}
@@ -755,67 +916,24 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
{
s32 cpu = cpu_of(rq);
u64 missing;
/* migration-disabled tasks and the rescuee are admitted capless */
if (is_migration_disabled(p) || p == scx_rescuee(rq))
return false;
missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
/* check caps against the sched running a bypassed task */
missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu,
scx_caps_for_task(p));
if (likely(!missing))
return false;
p->scx.reenq_reason_caps = missing;
p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));
p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu);
return true;
}
/*
* Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
* from p->scx.reenq_reason_*.
*
* A task can be re-rejected repeatedly. The reenqueue is bounded per task in
* scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT.
* Rejection can't happen for root.
*/
void scx_reenq_reject(struct rq *rq)
{
LIST_HEAD(tasks);
struct task_struct *p, *n;
lockdep_assert_rq_held(rq);
if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
return;
/*
* Move to a private list so a task re-rejected by the
* scx_do_enqueue_task() below isn't revisited this round.
*/
list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
/* migration_pending tasks should have bypassed to local DSQ */
if (WARN_ON_ONCE(p->migration_pending))
continue;
scx_wait_dispatching(p);
scx_dispatch_dequeue(rq, p);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
p->scx.flags |= SCX_TASK_REENQ_CAP;
list_add_tail(&p->scx.dsq_list.node, &tasks);
}
list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
list_del_init(&p->scx.dsq_list.node);
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
}
}
/* record a caps change, see struct scx_caps_updated */
static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps,
struct list_head *to_deliver)
@@ -921,7 +1039,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
/*
* Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check
* Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check
* below sees the node off the list and queues it, or the in-flight sync
* sees the caps[] update made before this call.
*/
@@ -946,7 +1064,7 @@ static void discard_queued_syncs(struct rq *rq)
}
/**
* scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
* @rq: the cid's cpu rq
* @prev: @rq's previous task from the in-progress dispatch
*
@@ -958,16 +1076,14 @@ static void discard_queued_syncs(struct rq *rq)
* learns the cid's idle state. Such a gain arms the per-rq
* %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
*/
void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
{
s32 cpu = cpu_of(rq);
s32 cid, shard;
struct llist_node *batch, *pos, *tmp;
u64 lost_all = 0;
lockdep_assert_rq_held(rq);
if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
return;
/*
@@ -1008,27 +1124,43 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
lost_all |= lost;
/*
* Tell the sched its effective caps on this cid changed. The
* invocation is equivalent to the dispatch path and may drop
* and re-acquire the rq lock temporarily while the rest of
* @batch is held privately, see scx_discard_ecaps_to_sync().
* The dispatch kfuncs resolve their context on the executing
* cpu, which under core scheduling can differ from @rq's cpu,
* so the context is set up there. The rq recorded in it keeps
* the dispatches targeting @rq.
* Tell the sched and its parent that the sched's effective caps
* on this cid changed. The invocations are equivalent to the
* dispatch path and may drop and re-acquire the rq lock
* temporarily while the rest of @batch is held privately, see
* scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve
* their context on the executing cpu, which under core
* scheduling can differ from @rq's cpu, so the context is set
* up there. The rq recorded in it keeps the dispatches
* targeting @rq.
*
* Bypass propagates down the hierarchy, so a sched that isn't
* bypassing has no bypassing parent. The child's bypass state
* gates both deliveries. Both report the same before value, so
* one reported_ecaps covers them.
*/
if (ecaps != pcpu->reported_ecaps &&
SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
!scx_bypassing(pcpu->sch, cpu)) {
struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) {
struct scx_sched *parent = scx_parent(pcpu->sch);
struct scx_dsp_ctx *dspc;
dspc->rq = rq;
/* stash @prev so nested dispatches can access it */
rq->scx.sub_dispatch_prev = prev;
SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
pcpu->reported_ecaps, ecaps);
if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) {
dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx;
dspc->rq = rq;
SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq,
scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps);
scx_flush_dispatch_buf(pcpu->sch, rq);
}
if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) {
dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx;
dspc->rq = rq;
SCX_CALL_OP(parent, sub_child_ecaps_updated, rq,
pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu),
pcpu->reported_ecaps, ecaps);
scx_flush_dispatch_buf(parent, rq);
}
rq->scx.sub_dispatch_prev = NULL;
scx_flush_dispatch_buf(pcpu->sch, rq);
pcpu->reported_ecaps = ecaps;
}
@@ -1153,6 +1285,34 @@ void scx_offline_ecaps(struct rq *rq)
}
}
/*
* Clear every cap @sch holds. The pshard caps go first as they are the source a
* pending sync recomputes ecaps from. ecaps are then zeroed directly for the
* cap checks.
*/
static void clear_all_caps(struct scx_sched *sch)
{
s32 si, cpu;
u32 cap_bit;
/* enable may have failed before scx_alloc_pshards() */
if (!sch->pshard)
return;
for (si = 0; si < sch->nr_pshards; si++) {
struct scx_pshard *ps = sch->pshard[si];
guard(raw_spinlock_irqsave)(&ps->lock);
for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++)
scx_cmask_clear(&ps->caps[cap_bit].cmask);
}
for_each_possible_cpu(cpu) {
guard(rq_lock_irqsave)(cpu_rq(cpu));
WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0);
}
}
/*
* @pcpu's sched was unhashed before the grace period, so nothing re-queues its
* sync node. Remove the node from @rq's pending list so the pcpu can be freed.
@@ -1252,7 +1412,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p)
lockdep_assert_held(&p->pi_lock);
lockdep_assert_rq_held(task_rq(p));
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
scx_prepare_task_sched_change(p);
scoped_guard (sched_change, p,
DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
scx_disable_and_exit_task(scx_task_sched(p), p);
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_set_task_state(p, SCX_TASK_INIT);
@@ -1282,7 +1444,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p)
lockdep_assert_rq_held(task_rq(p));
WARN_ON_ONCE(!READ_ONCE(to->bypass_depth));
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
scx_prepare_task_sched_change(p);
scoped_guard (sched_change, p,
DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
scx_disable_and_exit_task(scx_task_sched(p), p);
scx_set_task_sched(p, to);
}
@@ -1358,7 +1522,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
.sched_idle = tg->scx.idle,
.sched_idle = tg->scx.sched_idle,
};
if (tg->scx.sched != parent ||
@@ -1462,7 +1626,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
.sched_idle = tg->scx.idle,
.sched_idle = tg->scx.sched_idle,
};
/* the first pass must have transferred everything */
@@ -1616,6 +1780,8 @@ dump:
scx_cgroup_unlock();
percpu_up_write(&scx_fork_rwsem);
scx_ops_cid_sched_updated_disable(sch);
/*
* All tasks are moved off of @sch but there may still be on-going
* operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use
@@ -1628,6 +1794,12 @@ dump:
scx_unlink_sched(sch);
/*
* The parent restores what it delegated in ops.sub_detach(). @sch must
* hold no caps by then.
*/
clear_all_caps(sch);
mutex_unlock(&scx_enable_mutex);
/*
@@ -1787,6 +1959,15 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (ret)
goto err_disable;
/*
* Bypass before @sch is linked and grants can reach it. The parent's
* delivery advances reported_ecaps, so a sync consumed before @sch's
* ops are registered would reach the parent and never be replayed to
* @sch. While bypassed, the syncs are consumed without a delivery and
* replayed to both at unbypass.
*/
scx_bypass(sch, true);
ret = scx_link_sched(sch);
if (ret)
goto err_disable;
@@ -1832,12 +2013,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
}
sch->sub_attached = true;
scx_bypass(sch, true);
for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
if (((void (**)(void))ops)[i])
set_bit(i, sch->has_op);
scx_ops_cid_sched_updated_enable(sch);
percpu_down_write(&scx_fork_rwsem);
scx_cgroup_lock();
@@ -1940,7 +2121,9 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (!(p->scx.flags & SCX_TASK_SUB_INIT))
continue;
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
scx_prepare_task_sched_change(p);
scoped_guard (sched_change, p,
DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
/*
* $p must be either READY or ENABLED. If ENABLED,
* __scx_disabled_and_exit_task() first disables and
@@ -2266,6 +2449,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *
if (unlikely(!parent))
return -ENODEV;
if (!scx_kf_allowed_ctx(parent))
return -EDEADLK;
if (!scx_is_cid_type()) {
scx_error(parent, "sub-cap kfuncs require a cid-form scheduler");
return -EOPNOTSUPP;
@@ -2306,8 +2492,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *
* All-or-nothing keeps the caller-visible result binary per cid, so the denied
* mask is one mask to interpret rather than a per-cap matrix.
*
* Return 0 on full success, -EPERM if any cid was refused, or a negative
* errno on other failures.
* Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child
* is gone or being disabled, or a negative errno on other failures.
*/
__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
const struct scx_cmask *cmask__arena,
@@ -2361,6 +2547,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
scoped_guard (raw_spinlock, &pps->lock) {
guard(raw_spinlock_nested)(&cps->lock);
/* the child is being disabled, see clear_all_caps() */
if (unlikely(READ_ONCE(child->aborting))) {
ret = -ENODEV;
goto out;
}
/*
* Narrow granted_cids to cids the parent holds every
* requested cap on. All-or-nothing per cid.
@@ -2413,9 +2605,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
}
}
ret = any_denied ? -EPERM : 0;
out:
caps_updated_deliver(&to_deliver);
return any_denied ? -EPERM : 0;
return ret;
}
/**
kernel/sched/ext/sub.h
+39 -11
View File
@@ -16,6 +16,9 @@
struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch);
void scx_ops_cid_sched_updated_enable(struct scx_sched *sch);
void scx_ops_cid_sched_updated_disable(struct scx_sched *sch);
void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch);
struct cgroup *sch_cgroup(struct scx_sched *sch);
void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch);
@@ -28,16 +31,15 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
void scx_free_pshards(struct scx_sched *sch);
s32 scx_alloc_pshards(struct scx_sched *sch);
void scx_init_root_caps(struct scx_sched *sch);
void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
void scx_discard_stale_ecaps_syncs(void);
struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags);
struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags);
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
void scx_reenq_reject(struct rq *rq);
void scx_rescue_charge(struct rq *rq, s64 delta_exec);
void scx_rescue_end(struct rq *rq);
bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
@@ -72,10 +74,32 @@ static inline void scx_dec_has_subs(struct scx_sched *sch)
static_branch_dec(&__scx_has_subs);
}
/* hot-path hooks, gated inline so that a root-only system skips the calls */
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
{
lockdep_assert_rq_held(rq);
if (scx_has_subs())
__scx_process_sync_ecaps(rq, prev);
}
static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch,
struct rq *rq,
struct task_struct *p,
u64 *enq_flags)
{
if (!scx_has_subs())
return &rq->scx.local_dsq;
return __scx_resolve_local_dsq(sch, rq, p, enq_flags);
}
#else /* CONFIG_EXT_SUB_SCHED */
static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; }
static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {}
static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {}
static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {}
static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; }
@@ -93,7 +117,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp
static inline void scx_discard_stale_ecaps_syncs(void) {}
static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_reenq_reject(struct rq *rq) {}
static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
static inline void scx_rescue_end(struct rq *rq) {}
static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
@@ -169,17 +192,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p)
return SCX_CAP_ENQ;
}
/* the cap @sch needs to preempt @rq's current task, 0 if none */
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq,
u64 enq_flags)
{
struct task_struct *curr = rq->curr;
struct task_struct *donor = rq->donor;
/* a kernel-forced placement preempts regardless of caps */
if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
return 0;
/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
if (curr->sched_class != &ext_sched_class ||
scx_is_descendant(scx_task_sched(curr), sch))
if (donor->sched_class != &ext_sched_class ||
scx_is_descendant(scx_task_sched(donor), sch))
return 0;
return SCX_CAP_PREEMPT;
}
@@ -199,6 +223,8 @@ static inline u64 scx_caps_implied(u64 cap)
/* may @p keep running on @rq's cpu? requires baseline cpu access */
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
{
s32 cpu = cpu_of(rq);
if (!scx_has_subs())
return true;
@@ -206,7 +232,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p
if (unlikely(is_migration_disabled(p)))
return true;
return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
/* check caps against the sched running a bypassed task */
return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu,
SCX_CAP_BASE));
}
/* the task admitted for rescue on @rq, NULL if none */
kernel/sched/ext/types.h
+23 -5
View File
@@ -59,17 +59,31 @@ enum scx_consts {
};
/*
* Per-cid topology info. For each topology level (core, LLC, node) and shard,
* records the first cid in the unit and its global index. Global indices are
* consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
* [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
* set to -1 but always have valid shard assignments.
* Per-cid topology info. For each topology level (core, cluster, LLC, node) and
* shard, records the first cid in the unit and its global index. Global indices
* are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
* over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/
* node fields set to -1 but always have valid shard assignments.
*
* A cluster is the cache-sharing level between the core and the LLC. Where
* this level is absent, each core forms a cluster of its own. Where the cache
* level spans the LLC, the whole LLC forms one cluster. Each cluster has a
* unique cluster_cid and a dense cluster_idx, and its cids form a contiguous
* range.
*
* Every cid therefore has a cluster, so cluster_cid alone does not say whether
* the machine has the level: the first cluster of an LLC starts at the LLC's
* own base cid and a core-wide one at the core's.
*
* Shards are contiguous CID ranges used as scalable locking/work domains for
* sub-scheduler operations. By default each LLC becomes one shard, split into
* smaller shards if the LLC exceeds the target size. No-topo cids are packed
* into their own max-sized shards.
*
* Shards are cut on core boundaries only, so a shard doesn't need to contain a
* whole cluster: a cluster may straddle two shards. Only core, cluster, LLC
* and node are nested, in that order.
*
* New fields are appended, never inserted: scx_bpf_cid_topo() copies this
* struct out sized by the program's own layout, and an older program's copy
* must stay a prefix of the kernel's.
@@ -82,6 +96,8 @@ enum scx_consts {
* @node_idx: global index of that node, in [0, nr_nodes_at_init)
* @shard_cid: first cid of this cid's shard
* @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
* @cluster_cid: first cid of this cid's cluster
* @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init)
*/
struct scx_cid_topo {
s32 core_cid;
@@ -92,6 +108,8 @@ struct scx_cid_topo {
s32 node_idx;
s32 shard_cid;
s32 shard_idx;
s32 cluster_cid;
s32 cluster_idx;
};
enum scx_cid_consts {
kernel/sched/sched.h
+10 -1
View File
@@ -791,6 +791,10 @@ enum scx_rq_flags {
SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */
SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */
SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */
SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */
SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */
SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */
SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */
SCX_RQ_IN_WAKEUP = 1 << 16,
SCX_RQ_IN_DISPATCH = 1 << 17,
@@ -810,8 +814,8 @@ struct scx_rq_rescue {
struct scx_rq {
struct scx_dispatch_q local_dsq;
struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */
struct scx_rq_rescue rescue;
#endif
struct list_head runnable_list; /* runnable tasks on this rq */
@@ -830,6 +834,8 @@ struct scx_rq {
#endif
u64 clock; /* current per-rq clock -- see scx_bpf_now() */
#ifdef CONFIG_EXT_SUB_SCHED
/* sched of the open running session, see scx_cid_sched_update() */
struct scx_sched *sched;
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
struct task_struct *sub_dispatch_prev;
#endif
@@ -844,6 +850,9 @@ struct scx_rq {
struct list_head deferred_reenq_users; /* user DSQs requesting reenq */
struct balance_callback deferred_bal_cb;
struct balance_callback kick_sync_bal_cb;
#ifdef CONFIG_NO_HZ_FULL
struct balance_callback proxy_tick_bal_cb;
#endif
struct irq_work deferred_irq_work;
struct irq_work kick_cpus_irq_work;
};
kernel/sched/syscalls.c
+2
View File
@@ -678,6 +678,8 @@ change:
if (prev_class != next_class)
queue_flags |= DEQUEUE_CLASS;
scx_prepare_setscheduler(p, policy);
scoped_guard (sched_change, p, queue_flags) {
if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
tools/sched_ext/Makefile
+1 -1
View File
@@ -89,7 +89,7 @@ ifneq ($(wildcard $(GENHDR)),)
GENFLAGS := -DHAVE_GENHDR
endif
CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \
CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \
-I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
-I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include
tools/sched_ext/include/scx/cid.bpf.h
+599 -98
View File
@@ -3,8 +3,8 @@
* BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the
* authoritative layout and semantics. The BPF-side helpers use the cmask_*
* naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is
* redundant in BPF code. Atomics use __sync_val_compare_and_swap and every
* helper is inline (no .c counterpart).
* redundant in BPF code. Every helper is inline except the binary operations,
* which are weak global functions defined here, so there is no .c counterpart.
*
* Included by scx/common.bpf.h; don't include directly.
*
@@ -16,6 +16,11 @@
#include "bpf_arena_common.bpf.h"
/* libbpf's bpf_helpers.h defines it from 1.4 on */
#ifndef __arg_arena
#define __arg_arena __attribute((btf_decl_tag("arg:arena")))
#endif
#ifndef BIT_U64
#define BIT_U64(nr) (1ULL << (nr))
#endif
@@ -76,7 +81,7 @@ static __always_inline void __cmask_init(struct scx_cmask __arena *m, u32 base,
m->nr_cids = nr_cids;
m->alloc_words = alloc_words;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= alloc_words)
break;
m->bits[i] = 0;
@@ -122,24 +127,115 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base,
m->nr_cids = nr_cids;
}
/**
* cmask_nr_words - Words of bits[] that @m's active range spans
* @m: cmask to measure
*
* @m->base need not be word aligned: bits[0] covers the whole word @m->base
* falls in, so the count is taken from that word, not from @m->base.
*/
static __always_inline u32 cmask_nr_words(const struct scx_cmask __arena *m)
{
u32 wbase = m->base / 64;
return m->nr_cids ? (m->base + m->nr_cids - 1) / 64 - wbase + 1 : 0;
}
/**
* cmask_word - Read word @k of @m
* @m: cmask to read
* @k: word index, counted from the word @m->base falls in
*
* A word past the active range reads as 0, so a scan that looks one word
* ahead (at an SMT sibling that falls in the next word, say) needs no bound
* of its own.
*/
static __always_inline u64 cmask_word(const struct scx_cmask __arena *m, u32 k)
{
if (k >= cmask_nr_words(m))
return 0;
return m->bits[k];
}
/**
* cmask_range_word - Bits of word @k of @m that fall in [@start, @start + @nr)
* @m: cmask the word belongs to
* @k: word index, counted from the word @m->base falls in
* @start: first cid of the range
* @nr: number of cids in the range
*
* Scoping a scan to a domain that is contiguous in cid space, an LLC or a
* node, is then one AND per word, rather than a test of the ends of the
* range at every bit or a mask kept per domain.
*/
static __always_inline u64 cmask_range_word(const struct scx_cmask __arena *m,
u32 k, u32 start, u32 nr)
{
u64 wlo = (u64)(m->base / 64 + k) * 64, whi = wlo + 64;
u64 lo = start, hi = (u64)start + nr;
if (lo < wlo)
lo = wlo;
if (hi > whi)
hi = whi;
if (lo >= hi)
return 0;
return GENMASK_U64(hi - wlo - 1, lo - wlo);
}
/**
* __cmask_test - Test a cid without checking it against the active range
* @cid: cid to test
* @m: cmask to test
*
* The caller must already know that @cid lies within
* [@m->base, @m->base + @m->nr_cids), or the read runs off @m->bits.
*
* For hot scans that have already bounded @cid, where the per-call range check
* is repeated work on every candidate.
*/
static __always_inline bool __cmask_test(u32 cid, const struct scx_cmask __arena *m)
{
return *__cmask_word(cid, m) & BIT_U64(cid & 63);
}
static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m)
{
if (!__cmask_contains(cid, m))
return false;
return *__cmask_word(cid, m) & BIT_U64(cid & 63);
return __cmask_test(cid, m);
}
/*
* x86 BPF JIT rejects BPF_OR | BPF_FETCH and BPF_AND | BPF_FETCH on arena
* pointers (see bpf_jit_supports_insn() in arch/x86/net/bpf_jit_comp.c). Only
* BPF_CMPXCHG / BPF_XCHG / BPF_ADD with FETCH are allowed. Implement
* test_and_{set,clear} and the atomic set/clear via a cmpxchg loop.
* The bit helpers use the fetching bitwise builtins, which order like the
* cmpxchg they replace, and not every JIT accepts those on arena pointers. The
* loader sets SCX_LIB_FEAT_ARENA_FETCH_BITOPS when a probe with the fetching
* form loads, and the helpers run a cmpxchg loop otherwise. The fetched value
* is consumed through barrier_var() because clang 19 lowers a builtin whose
* result is unused to the non-fetching form, which is relaxed on arm64.
*
* CMASK_CAS_TRIES is sized so exhausting it means seconds of real spinning
* on one word - past any plausible contention. Abort hard.
* on one word, past any plausible contention. Abort hard.
*/
#define CMASK_CAS_TRIES (1U << 23)
static __always_inline u64 __cmask_fetch_or(u64 __arena *w, u64 mask)
{
u64 old = __atomic_fetch_or(w, mask, __ATOMIC_ACQ_REL);
barrier_var(old);
return old;
}
static __always_inline u64 __cmask_fetch_andnot(u64 __arena *w, u64 mask)
{
u64 old = __atomic_fetch_and(w, ~mask, __ATOMIC_ACQ_REL);
barrier_var(old);
return old;
}
static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m)
{
u64 __arena *w;
@@ -150,7 +246,11 @@ static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m)
return;
w = __cmask_word(cid, m);
bit = BIT_U64(cid & 63);
bpf_for(i, 0, CMASK_CAS_TRIES) {
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) {
__cmask_fetch_or(w, bit);
return;
}
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if (old & bit)
return;
@@ -171,7 +271,11 @@ static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m)
return;
w = __cmask_word(cid, m);
bit = BIT_U64(cid & 63);
bpf_for(i, 0, CMASK_CAS_TRIES) {
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) {
__cmask_fetch_andnot(w, bit);
return;
}
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if (!(old & bit))
return;
@@ -192,7 +296,9 @@ static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena
return false;
w = __cmask_word(cid, m);
bit = BIT_U64(cid & 63);
bpf_for(i, 0, CMASK_CAS_TRIES) {
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS))
return __cmask_fetch_or(w, bit) & bit;
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if (old & bit)
return true;
@@ -214,7 +320,9 @@ static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __are
return false;
w = __cmask_word(cid, m);
bit = BIT_U64(cid & 63);
bpf_for(i, 0, CMASK_CAS_TRIES) {
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS))
return __cmask_fetch_andnot(w, bit) & bit;
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if (!(old & bit))
return false;
@@ -268,17 +376,242 @@ static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __a
return prev;
}
/* atomically or @mask into *@w */
static __always_inline void __cmask_word_or(u64 __arena *w, u64 mask)
{
u64 old, new;
u32 i;
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) {
__cmask_fetch_or(w, mask);
return;
}
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if ((old & mask) == mask)
return;
new = old | mask;
if (__sync_val_compare_and_swap(w, old, new) == old)
return;
}
scx_bpf_error("__cmask_word_or CAS exhausted");
}
/* atomically clear @mask bits in *@w */
static __always_inline void __cmask_word_andnot(u64 __arena *w, u64 mask)
{
u64 old, new;
u32 i;
if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) {
__cmask_fetch_andnot(w, mask);
return;
}
bpf_arena_for(i, 0, CMASK_CAS_TRIES) {
old = *w;
if (!(old & mask))
return;
new = old & ~mask;
if (__sync_val_compare_and_swap(w, old, new) == old)
return;
}
scx_bpf_error("__cmask_word_andnot CAS exhausted");
}
/**
* cmask_full_range - Test whether every cid in [@start, @start + @nr) is set
* @m: cmask to test
* @start: first cid of the range
* @nr: number of cids in the range
*
* The range is clamped to @m's active range first. True when no clamped bit is
* clear, including when the clamped range is empty.
*/
static __always_inline bool cmask_full_range(const struct scx_cmask __arena *m, u32 start,
u32 nr)
{
u64 end = (u64)start + nr;
u32 wbase = m->base / 64;
u32 first_wi, last_wi, first_bit, last_bit, last, i;
if (start < m->base)
start = m->base;
if (end > m->base + m->nr_cids)
end = m->base + m->nr_cids;
if (start >= end)
return true;
last = end - 1;
first_wi = start / 64 - wbase;
last_wi = last / 64 - wbase;
first_bit = start & 63;
last_bit = last & 63;
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 wi = first_wi + i;
u64 mask = ~0ULL;
if (wi > last_wi)
break;
if (wi == first_wi)
mask &= GENMASK_U64(63, first_bit);
if (wi == last_wi)
mask &= GENMASK_U64(last_bit, 0);
if ((m->bits[wi] & mask) != mask)
return false;
}
return true;
}
/**
* cmask_set_range - Set every cid in [@start, @start + @nr)
* @m: cmask to modify
* @start: first cid of the range
* @nr: number of cids in the range
*
* The range is clamped to @m's active range first. Words are updated atomically
* so concurrent updates of other bits sharing a word are never lost. The range
* as a whole does not transition atomically.
*/
static __always_inline void cmask_set_range(struct scx_cmask __arena *m, u32 start, u32 nr)
{
u64 end = (u64)start + nr;
u32 wbase = m->base / 64;
u32 first_wi, last_wi, first_bit, last_bit, last, i;
if (start < m->base)
start = m->base;
if (end > m->base + m->nr_cids)
end = m->base + m->nr_cids;
if (start >= end)
return;
last = end - 1;
first_wi = start / 64 - wbase;
last_wi = last / 64 - wbase;
first_bit = start & 63;
last_bit = last & 63;
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 wi = first_wi + i;
u64 mask = ~0ULL;
if (wi > last_wi)
break;
if (wi == first_wi)
mask &= GENMASK_U64(63, first_bit);
if (wi == last_wi)
mask &= GENMASK_U64(last_bit, 0);
__cmask_word_or(&m->bits[wi], mask);
}
}
/**
* cmask_clear_range - Clear every cid in [@start, @start + @nr)
* @m: cmask to modify
* @start: first cid of the range
* @nr: number of cids in the range
*
* The range is clamped to @m's active range first. Words are updated atomically
* so concurrent updates of other bits sharing a word are never lost. The range
* as a whole does not transition atomically.
*/
static __always_inline void cmask_clear_range(struct scx_cmask __arena *m, u32 start, u32 nr)
{
u64 end = (u64)start + nr;
u32 wbase = m->base / 64;
u32 first_wi, last_wi, first_bit, last_bit, last, i;
if (start < m->base)
start = m->base;
if (end > m->base + m->nr_cids)
end = m->base + m->nr_cids;
if (start >= end)
return;
last = end - 1;
first_wi = start / 64 - wbase;
last_wi = last / 64 - wbase;
first_bit = start & 63;
last_bit = last & 63;
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 wi = first_wi + i;
u64 mask = ~0ULL;
if (wi > last_wi)
break;
if (wi == first_wi)
mask &= GENMASK_U64(63, first_bit);
if (wi == last_wi)
mask &= GENMASK_U64(last_bit, 0);
__cmask_word_andnot(&m->bits[wi], mask);
}
}
static __always_inline void cmask_zero(struct scx_cmask __arena *m)
{
u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
m->bits[i] = 0;
}
}
/**
* cmask_fill - Set every cid in @m's active range
* @m: cmask to fill
*
* Counterpart to cmask_zero(). Storage past the active range is left as is,
* matching the kernel-side scx_cmask_fill().
*/
static __always_inline void cmask_fill(struct scx_cmask __arena *m)
{
u32 nr_words, head_bits, tail_bits, i;
if (!m->nr_cids)
return;
nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
m->bits[i] = ~0LLU;
}
/* clear word-0 bits below base */
head_bits = m->base & 63;
if (head_bits)
m->bits[0] &= ~((1LLU << head_bits) - 1);
/* clear last-word bits at or past base + nr_cids */
tail_bits = (m->base + m->nr_cids) & 63;
if (tail_bits)
m->bits[nr_words - 1] &= (1LLU << tail_bits) - 1;
}
/**
* cmask_empty - Test whether no cid is set in @m's active range
* @m: cmask to test
*
* Scans the words the active range spans, whose bits outside the range every
* mutator keeps zero.
*/
static __always_inline bool cmask_empty(const struct scx_cmask __arena *m)
{
u32 nr_words = cmask_nr_words(m), i;
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
if (m->bits[i])
return false;
}
return true;
}
/*
* BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_*
* enum in ext/cid.c, which is exported via BTF and reachable through
@@ -291,92 +624,136 @@ enum {
BPF_CMASK_OP_ANDNOT,
};
static __always_inline void cmask_op_word(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src,
u32 di, u32 si, u64 mask, int op)
/* range bits of word @k of @m, which spans @nr words: 0 past the span */
static __always_inline u64 __cmask_word_range(const struct scx_cmask __arena *m, u32 k,
u32 nr)
{
u64 dv = dst->bits[di];
u64 sv = src->bits[si];
u64 rv;
u64 bits = ~0ULL;
if (op == BPF_CMASK_OP_AND)
rv = dv & sv;
else if (op == BPF_CMASK_OP_OR)
rv = dv | sv;
else if (op == BPF_CMASK_OP_ANDNOT)
rv = dv & ~sv;
else
rv = sv;
dst->bits[di] = (dv & ~mask) | (rv & mask);
}
static __always_inline void cmask_op(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src, int op)
{
u32 d_end = dst->base + dst->nr_cids;
u32 s_end = src->base + src->nr_cids;
u32 lo = dst->base > src->base ? dst->base : src->base;
u32 hi = d_end < s_end ? d_end : s_end;
u32 d_base = dst->base / 64;
u32 s_base = src->base / 64;
u32 lo_word, hi_word, i;
u64 head_mask, tail_mask;
if (lo >= hi)
return;
lo_word = lo / 64;
hi_word = (hi - 1) / 64;
head_mask = GENMASK_U64(63, lo & 63);
tail_mask = GENMASK_U64((hi - 1) & 63, 0);
bpf_for(i, 0, CMASK_MAX_WORDS) {
u32 w = lo_word + i;
u64 m;
if (w > hi_word)
break;
m = GENMASK_U64(63, 0);
if (w == lo_word)
m &= head_mask;
if (w == hi_word)
m &= tail_mask;
cmask_op_word(dst, src, w - d_base, w - s_base, m, op);
}
if (k >= nr)
return 0;
if (k == 0)
bits &= GENMASK_U64(63, m->base & 63);
if (k == nr - 1)
bits &= GENMASK_U64((m->base + m->nr_cids - 1) & 63, 0);
return bits;
}
/*
* cmask_and/or/copy only modify @dst bits that lie in the intersection of
* [@dst->base, @dst->base + @dst->nr_cids) and [@src->base,
* @src->base + @src->nr_cids). Bits in @dst outside that window
* keep their prior values - in particular, cmask_copy() does NOT zero @dst
* bits that lie outside @src's range.
* The binary operations combine cmasks of any active ranges. A cmask holds bits
* only for its own range and reads as the operation's identity outside it, all
* ones for AND and all zeros for OR, so an operand alters only the bits it
* covers: ANDing a shard's mask into a global one edits that shard's window and
* leaves the rest of the global mask alone. ANDNOT is AND with @src2 inverted
* over @src2's range, so where only @src2 has range the result is ~@src2.
*
* @dst is written on its range intersected with the union of the sources'
* ranges and untouched elsewhere, and its padding bits outside its own range
* stay zero, the invariant every cmask mutator keeps. @dst may alias either
* source, and the two sources may be one mask. Returns whether @dst has a bit
* set afterwards, over all of @dst's range.
*
* The operations are global functions, verified once per program. Inlined,
* every data-dependent branch in this loop would multiply the states the
* verifier explores per iteration at every call site.
*/
static __always_inline void cmask_and(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src)
static __always_inline bool __cmask_op(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src1,
const struct scx_cmask __arena *src2, int op)
{
cmask_op(dst, src, BPF_CMASK_OP_AND);
u32 d_first = dst->base / 64, d_nr = cmask_nr_words(dst);
u32 s1_first = src1->base / 64, s1_nr = cmask_nr_words(src1);
u32 s2_first = src2->base / 64, s2_nr = cmask_nr_words(src2);
u64 any = bpf_arena_loop_zero;
u32 i;
bpf_arena_for(i, 0, d_nr) {
u32 k1 = d_first + i - s1_first, k2 = d_first + i - s2_first;
u64 m1 = __cmask_word_range(src1, k1, s1_nr);
u64 m2 = op == BPF_CMASK_OP_COPY ? m1 : __cmask_word_range(src2, k2, s2_nr);
u64 um = __cmask_word_range(dst, i, d_nr) & (m1 | m2);
u64 dv = dst->bits[i], v1, v2, rv;
if (um) {
/* a word in range holds zeros outside the range */
v1 = m1 ? src1->bits[k1] : 0;
v2 = m2 ? src2->bits[k2] : 0;
if (op == BPF_CMASK_OP_AND)
rv = (v1 | ~m1) & (v2 | ~m2);
else if (op == BPF_CMASK_OP_OR)
rv = v1 | v2;
else if (op == BPF_CMASK_OP_ANDNOT)
rv = (v1 | ~m1) & ~v2;
else
rv = v1;
dv = (dv & ~um) | (rv & um);
dst->bits[i] = dv;
}
any |= dv;
}
return any != 0;
}
static __always_inline void cmask_or(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src)
/**
* cmask_and - Store @src1 AND @src2 in @dst
* @dst: destination, may be @src1 or @src2
* @src1: first operand
* @src2: second operand
*
* See __cmask_op() for how the ranges combine. Return whether @dst has a bit
* set afterwards.
*/
__weak bool cmask_and(struct scx_cmask __arena __arg_arena *dst,
const struct scx_cmask __arena __arg_arena *src1,
const struct scx_cmask __arena __arg_arena *src2)
{
cmask_op(dst, src, BPF_CMASK_OP_OR);
return __cmask_op(dst, src1, src2, BPF_CMASK_OP_AND);
}
static __always_inline void cmask_copy(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src)
/**
* cmask_or - Store @src1 OR @src2 in @dst
* @dst: destination, may be @src1 or @src2
* @src1: first operand
* @src2: second operand
*
* See __cmask_op() for how the ranges combine. Return whether @dst has a bit
* set afterwards.
*/
__weak bool cmask_or(struct scx_cmask __arena __arg_arena *dst,
const struct scx_cmask __arena __arg_arena *src1,
const struct scx_cmask __arena __arg_arena *src2)
{
cmask_op(dst, src, BPF_CMASK_OP_COPY);
return __cmask_op(dst, src1, src2, BPF_CMASK_OP_OR);
}
static __always_inline void cmask_andnot(struct scx_cmask __arena *dst,
const struct scx_cmask __arena *src)
/**
* cmask_andnot - Store @src1 AND NOT @src2 in @dst
* @dst: destination, may be @src1 or @src2
* @src1: operand to remove bits from
* @src2: bits to remove, inverted over its own range
*
* See __cmask_op() for how the ranges combine. Return whether @dst has a bit
* set afterwards.
*/
__weak bool cmask_andnot(struct scx_cmask __arena __arg_arena *dst,
const struct scx_cmask __arena __arg_arena *src1,
const struct scx_cmask __arena __arg_arena *src2)
{
cmask_op(dst, src, BPF_CMASK_OP_ANDNOT);
return __cmask_op(dst, src1, src2, BPF_CMASK_OP_ANDNOT);
}
/**
* cmask_copy - Copy @src into @dst
* @dst: destination
* @src: source
*
* The single-source case of __cmask_op(): @dst is written over @src's range and
* untouched elsewhere.
*/
__weak void cmask_copy(struct scx_cmask __arena __arg_arena *dst,
const struct scx_cmask __arena __arg_arena *src)
{
__cmask_op(dst, src, src, BPF_CMASK_OP_COPY);
}
/*
@@ -395,7 +772,7 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
return true;
nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
if (a->bits[i] != b->bits[i])
@@ -428,7 +805,7 @@ static __always_inline u32 cmask_next_set(const struct scx_cmask __arena *m, u32
start_wi = cid / 64 - base;
start_bit = cid & 63;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 wi = start_wi + i;
u64 word;
u32 found;
@@ -457,7 +834,7 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
#define cmask_for_each(cid, m) \
for ((cid) = cmask_first_set(m); \
(cid) < (m)->base + (m)->nr_cids; \
(cid) < (m)->base + (m)->nr_cids && can_loop; \
(cid) = cmask_next_set((m), (cid) + 1))
/*
@@ -495,7 +872,7 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
lo_word = lo / 64;
hi_word = (hi - 1) / 64;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 w = lo_word + i;
if (w > hi_word)
@@ -514,13 +891,14 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
{
u32 nr_words, i;
u32 count = 0;
/* callers compare the sum, see bpf_arena_loop_zero */
u32 count = bpf_arena_loop_zero;
if (!m->nr_cids)
return 0;
nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
count += __builtin_popcountll(m->bits[i]);
@@ -528,10 +906,7 @@ static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
return count;
}
/*
* True if @a and @b share any set bit. Walk only the intersection of their
* ranges, matching the semantics of cmask_and().
*/
/* true if @a and @b share any set bit, over the intersection of their ranges */
static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b)
{
@@ -552,7 +927,7 @@ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a,
head_mask = GENMASK_U64(63, lo & 63);
tail_mask = GENMASK_U64((hi - 1) & 63, 0);
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 w = lo_word + i;
u64 mask, av, bv;
@@ -603,7 +978,7 @@ static __always_inline u32 cmask_next_and_set(const struct scx_cmask __arena *a,
start_wi = start / 64;
start_bit = start & 63;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 abs_wi = start_wi + i;
u64 word;
u32 found;
@@ -646,6 +1021,17 @@ static __always_inline u32 cmask_next_set_wrap(const struct scx_cmask __arena *m
return found < start ? found : end;
}
/**
* cmask_end - One past the last cid of @m's active range
* @m: cmask of interest
*
* The scan helpers return this value when no set cid is found.
*/
static __always_inline u32 cmask_end(const struct scx_cmask __arena *m)
{
return m->base + m->nr_cids;
}
/*
* Find the next cid set in both @a and @b at or after @start, wrapping to
* @a->base if none found in the forward half. Return a->base + a->nr_cids
@@ -668,6 +1054,121 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren
return found < start ? found : a_end;
}
/*
* The distribute helpers rotate through one slot per cid, a cacheline each so
* picks on one CPU do not bounce the line of another. A missing allocation is a
* setup bug and aborts the scheduler. On a CPU outside the scheduler's cid
* space the helpers fall back to the first matching cid.
*
* TODO: The slots are one allocation without node placement. Move them to a
* per-cid arena allocator once the library has one, for node-local slots
* without the cacheline padding.
*/
struct cmask_rotor_slot {
u32 cid;
} __attribute__((aligned(SCX_CACHELINE_SIZE)));
struct cmask_rotor_slot __arena *cmask_distribute_rotors __weak;
/**
* cmask_distribute_init - Allocate the distribute rotor from @map
* @map: the scheduler's arena map
*
* The shared arena init calls this once. A scheduler with its own arena calls
* it before the first distribute pick. On a kernel without the cid kfuncs
* nothing can pick, so the call allocates nothing and succeeds. Return 0 on
* success, -ENOMEM if the allocation fails.
*/
static __always_inline int cmask_distribute_init(void *map)
{
u64 size;
u32 pages;
/* no cid kfuncs means no picks and nothing to allocate */
if (!bpf_ksym_exists(scx_bpf_nr_cids))
return 0;
size = scx_bpf_nr_cids() * sizeof(*cmask_distribute_rotors);
pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
cmask_distribute_rotors = bpf_arena_alloc_pages(map, NULL, pages, NUMA_NO_NODE, 0);
return cmask_distribute_rotors ? 0 : -ENOMEM;
}
static __always_inline u32 __arena *__cmask_distribute_rotor(void)
{
s32 cid = scx_bpf_this_cid();
if (unlikely(!cmask_distribute_rotors)) {
scx_bpf_error("cmask distribute rotor not allocated");
return NULL;
}
if (cid < 0)
return NULL;
return &cmask_distribute_rotors[cid].cid;
}
/**
* cmask_any_distribute - Pick a set cid, spreading successive picks
* @m: cmask to pick from
*
* Counterpart of bpf_cpumask_any_distribute(): a per-cid rotor makes successive
* picks rotate through the set cids instead of repeating the first one. Returns
* cmask_end(@m) if @m is empty.
*/
static __always_inline u32 cmask_any_distribute(const struct scx_cmask __arena *m)
{
u32 __arena *rotor = __cmask_distribute_rotor();
u32 pick;
if (unlikely(!rotor))
return cmask_first_set(m);
pick = cmask_next_set_wrap(m, *rotor + 1);
if (pick < cmask_end(m))
*rotor = pick;
return pick;
}
/**
* cmask_any_and_distribute - Pick a cid set in both masks, spreading picks
* @a: first cmask, the scan is bounded by its range
* @b: second cmask
*
* Counterpart of bpf_cpumask_any_and_distribute(). Shares the rotor with
* cmask_any_distribute() the same way the kernel counterparts share theirs.
* Returns cmask_end(@a) if the intersection is empty.
*/
static __always_inline u32 cmask_any_and_distribute(const struct scx_cmask __arena *a,
const struct scx_cmask __arena *b)
{
u32 __arena *rotor = __cmask_distribute_rotor();
u32 pick;
if (unlikely(!rotor))
return cmask_next_and_set(a, b, a->base);
pick = cmask_next_and_set_wrap(a, b, *rotor + 1);
if (pick < cmask_end(a))
*rotor = pick;
return pick;
}
/**
* cmask_distribute_rotor_pos - Last cid picked by the distribute helpers
*
* For callers that anchor scans of their own on the shared rotor. Returns 0
* when nothing has been picked on this cid yet or the CPU is outside the cid
* space.
*/
static __always_inline u32 cmask_distribute_rotor_pos(void)
{
u32 __arena *rotor = __cmask_distribute_rotor();
if (unlikely(!rotor))
return 0;
return *rotor;
}
/*
* Like cmask_next_and_set() but over the intersection of THREE masks. Return
* a->base + a->nr_cids if no cid is set in all three at or after @start.
@@ -701,7 +1202,7 @@ static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a
start_wi = start / 64;
start_bit = start & 63;
bpf_for(i, 0, CMASK_MAX_WORDS) {
bpf_arena_for(i, 0, CMASK_MAX_WORDS) {
u32 abs_wi = start_wi + i;
u64 word;
u32 found;
@@ -763,7 +1264,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m,
s32 cpu;
cmask_zero(m);
bpf_for(cpu, 0, nr_cpu_ids) {
bpf_arena_for(cpu, 0, nr_cpu_ids) {
s32 cid;
if (!bpf_cpumask_test_cpu(cpu, cpumask))
tools/sched_ext/include/scx/common.bpf.h
+86 -9
View File
@@ -27,6 +27,7 @@
#include "user_exit_info.bpf.h"
#include "enum_defs.autogen.h"
#include "bpf_arena_common.bpf.h"
#include "const-defs.h"
#define PF_IDLE 0x00000002 /* I am an IDLE thread */
#define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
@@ -74,6 +75,7 @@ struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak;
int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it, struct task_struct *p) __ksym __weak;
void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak;
@@ -104,10 +106,11 @@ struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak;
struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak;
u64 scx_bpf_now(void) __ksym __weak;
void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak;
u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) __ksym __weak;
s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak;
s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak;
void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak;
void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
s32 scx_bpf_cid_node(s32 cid) __ksym __weak;
/* scx_bpf_cid_topo() and scx_bpf_kick_cid() are declared in compat.bpf.h */
s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak;
s32 scx_bpf_this_cid(void) __ksym __weak;
struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
@@ -116,7 +119,7 @@ u32 scx_bpf_nr_online_cids(void) __ksym __weak;
const void __arena *scx_bpf_online_cmask(void) __ksym __weak;
u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
/* scx_bpf_cidperf_set() is declared in compat.bpf.h */
/* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */
s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak;
@@ -307,7 +310,7 @@ BPF_PROG(name, ##args)
* Similar to MEMBER_VPTR() but is intended for use with arrays where the
* element count needs to be explicit.
* It can be used in cases where a global array is defined with an initial
* size but is intended to be be resized before loading the BPF program.
* size but is intended to be resized before loading the BPF program.
* Without this version of the macro, MEMBER_VPTR() will use the compile time
* size of the array to compute the max, which will result in rejection by
* the verifier.
@@ -533,14 +536,14 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask)
/*
* True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls
* migrate_disable() for the current task. Recorded once by
* scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during
* the natural scheduler-attach call chain (auto-attached by scx_ops_attach!).
* scx_lib_init_probe, an fentry program that fires during the natural
* scheduler-attach call chain (auto-attached by scx_ops_attach!).
*
* Defaults to true (conservative). Over-reporting in is_migration_disabled()
* Defaults to false (conservative). Over-reporting in is_migration_disabled()
* causes local-only dispatch, which is safe. Under-reporting can crash the
* scheduler, so we err high if the probe somehow fails to run.
*/
bool __scx_prolog_disables_migration __weak = true;
bool __scx_prolog_disables_migration __weak = false;
/*
* scx_lib_init_probe - non-sleepable prolog probe.
@@ -551,13 +554,20 @@ bool __scx_prolog_disables_migration __weak = true;
* ops.init() fires. Its address is taken in the vtable, so the symbol
* is non-inlinable and has been stable since introduction.
*
* A cid-form scheduler registers through bpf_scx_reg_cid(), the .reg
* callback of bpf_sched_ext_ops_cid, and so never enters bpf_scx_reg().
* The cid-form open paths (SCX_OPS_CID_OPEN(), scx_ops_cid_open!())
* therefore repoint this program at bpf_scx_reg_cid(). Repointing rather
* than adding a second program keeps the object loadable on kernels
* predating the cid form, where bpf_scx_reg_cid() has no BTF entry.
*
* Entering via fentry runs us through __bpf_prog_enter -- the
* non-sleepable prolog that consumers of is_migration_disabled() live
* under.
*
* Loud warning: the prolog adds at most 1 to migration_disabled.
* Reading > 1 means something upstream in the
* bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration
* bpf_struct_ops_link_create -> .reg path disabled migration
* before the prolog ran, invalidating the probe; audit and adjust.
*/
SEC("fentry/bpf_scx_reg") __weak
@@ -1185,8 +1195,75 @@ static inline u64 scx_clock_irq(u32 cpu)
#define struct_size_t(type, member, count) \
struct_size((type *)NULL, member, count)
/*
* <linux/stddef.h>'s TRAILING_OVERLAP(): embed a struct that ends in a flexible
* array member together with its storage. @NAME is a complete @TYPE and the
* @MEMBERS declared after the array reserve the space the array grows into.
* Unlike the kernel's, the padding is named after @NAME so that one struct can
* embed several, and it is sized with __builtin_offsetof() because
* bpf_helpers.h redefines offsetof() as a pointer cast, which is not a constant
* expression.
*/
#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \
union { \
TYPE NAME; \
struct { \
unsigned char __offset_to_##NAME[__builtin_offsetof(TYPE, FAM)]; \
MEMBERS \
} ATTRS; \
}
#define TRAILING_OVERLAP(TYPE, NAME, FAM, MEMBERS) \
__TRAILING_OVERLAP(TYPE, NAME, FAM, /* no attrs */, MEMBERS)
/*
* Loop counters the verifier cannot see through.
*
* A may_goto loop converges when the state at its head is within the state
* of a previous iteration. A counter tracked as a precise constant prevents
* that convergence: every iteration creates a new state, and the verifier
* unrolls the loop until it runs out of budget whenever the counter feeds an
* operation that requires precision.
*
* Loading a writable global gives the verifier an unknown scalar while its
* runtime value remains one. Using it as the step makes the counter unknown
* after the first iteration, and the load on every increment keeps the loop
* body from making it precise again. volatile is required to keep the compiler
* from hoisting or eliminating the loads, and the value must remain non-const
* so the verifier cannot resolve it. The variable is weak so the objects linked
* into a scheduler share one copy. Its runtime value must never be changed from
* one.
*
* bpf_for() avoids this verifier behavior too, but calls bpf_iter_num_next()
* on every iteration. bpf_arena_for() is intended for hot scheduler walks
* where that cost matters. @var must be no wider than u32, and @start and
* @end must be representable as u32.
*/
volatile u32 bpf_arena_loop_one __weak = 1;
/*
* A loop-carried accumulator that a caller later compares is kept precise, and
* a precise scalar whose range grows every iteration keeps a may_goto loop from
* converging. Initializing it from this zero makes it unknown from the first
* iteration, so every pass through the loop head looks the same.
*/
volatile u32 bpf_arena_loop_zero __weak = 0;
#define __bpf_arena_loop_start(var, start) \
({ \
_Static_assert(sizeof(var) <= sizeof(u32), \
"bpf_arena_for() index must fit in u32"); \
(start); \
})
#define bpf_arena_for(var, start, end) \
for (var = __bpf_arena_loop_start(var, start); \
var < (end) && can_loop; \
var += bpf_arena_loop_one)
#include "compat.bpf.h"
#include "enums.bpf.h"
#include "features.bpf.h"
#include "cid.bpf.h"
#endif /* __SCX_COMMON_BPF_H */
tools/sched_ext/include/scx/compat.bpf.h
+90
View File
@@ -143,6 +143,69 @@ static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_t
shard_start, shard_start_cnt);
}
/*
* v7.3: b38332be61a8 ("sched_ext: Make scx_bpf_kick_cid() return void") flipped
* the return type from s32 to void without renaming the kfunc, so the v7.2 and
* v7.3 kfuncs share a name but have incompatible prototypes. libbpf leaves a
* weak kfunc unresolved when the prototype doesn't match, so declare both
* flavors: the ___vN suffix is stripped on the BPF side, both resolve against
* the same kernel symbol and only the matching one gets bound. Drop the
* wrapper and move the decl back to common.bpf.h after v7.6.
*/
void scx_bpf_kick_cid___v2(s32 cid, u64 flags) __ksym __weak;
s32 scx_bpf_kick_cid___v1(s32 cid, u64 flags) __ksym __weak;
static inline void scx_bpf_kick_cid(s32 cid, u64 flags)
{
if (bpf_ksym_exists(scx_bpf_kick_cid___v2))
scx_bpf_kick_cid___v2(cid, flags);
else if (bpf_ksym_exists(scx_bpf_kick_cid___v1))
scx_bpf_kick_cid___v1(cid, flags);
}
/*
* v7.3: 94480606a677 ("sched_ext: Add a size argument to scx_bpf_cid_topo() so
* struct scx_cid_topo can grow") added out__sz without renaming the kfunc, so
* the v7.2 and v7.3 kfuncs share a name with incompatible prototypes. Same
* two-flavor trick as scx_bpf_kick_cid() above. The size is always the output
* buffer's, so a macro supplies it and callers keep the two-argument form.
* Drop the wrapper and move the decl back to common.bpf.h after v7.6.
*/
void scx_bpf_cid_topo___v2(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak;
void scx_bpf_cid_topo___v1(s32 cid, struct scx_cid_topo *out) __ksym __weak;
static inline void __scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz)
{
if (bpf_ksym_exists(scx_bpf_cid_topo___v2))
scx_bpf_cid_topo___v2(cid, out, out__sz);
else if (bpf_ksym_exists(scx_bpf_cid_topo___v1))
scx_bpf_cid_topo___v1(cid, out);
}
#define scx_bpf_cid_topo(cid, out) __scx_bpf_cid_topo((cid), (out), sizeof(*(out)))
/*
* v7.3: 3a21e34eb258 ("sched_ext: Gate scx_bpf_cidperf_set() behind a new
* SCX_CAP_PERF") flipped the return type the other way, from void to s32, again
* without renaming the kfunc. Same two-flavor trick as scx_bpf_kick_cid()
* above; the v7.2 kfunc can't fail, so report success for it. Drop the wrapper
* and move the decl back to common.bpf.h after v7.6.
*/
s32 scx_bpf_cidperf_set___v2(s32 cid, u32 perf) __ksym __weak;
void scx_bpf_cidperf_set___v1(s32 cid, u32 perf) __ksym __weak;
static inline s32 scx_bpf_cidperf_set(s32 cid, u32 perf)
{
if (bpf_ksym_exists(scx_bpf_cidperf_set___v2)) {
return scx_bpf_cidperf_set___v2(cid, perf);
} else if (bpf_ksym_exists(scx_bpf_cidperf_set___v1)) {
scx_bpf_cidperf_set___v1(cid, perf);
return 0;
} else {
return -EOPNOTSUPP;
}
}
/**
* __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on
* in a compatible way. We will preserve this __COMPAT helper until v6.16.
@@ -243,6 +306,18 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \
scx_bpf_pick_any_cpu(cpus_allowed, flags))
/* v7.4: Add scx_bpf_cid_node(). */
#define __COMPAT_scx_bpf_cid_node(cid) \
(bpf_ksym_exists(scx_bpf_cid_node) ? \
scx_bpf_cid_node(cid) : NUMA_NO_NODE)
/*
* v7.4: Add scx_bpf_cgroup_nr_cpus().
*/
#define __COMPAT_scx_bpf_cgroup_nr_cpus(cgrp) \
(bpf_ksym_exists(scx_bpf_cgroup_nr_cpus) ? \
scx_bpf_cgroup_nr_cpus(cgrp) : scx_bpf_nr_cpu_ids())
/*
* v6.18: Add a helper to retrieve the current task running on a CPU.
*
@@ -403,6 +478,15 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime)
p->scx.dsq_vtime = vtime;
}
/* v7.4: Add scx_bpf_task_set_lazy_resched(). */
bool scx_bpf_task_set_lazy_resched___new(struct task_struct *p, bool lazy) __ksym __weak;
static inline void scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy)
{
if (bpf_ksym_exists(scx_bpf_task_set_lazy_resched___new))
scx_bpf_task_set_lazy_resched___new(p, lazy);
}
/*
* v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This
* will eventually deprecate scx_bpf_reenqueue_local().
@@ -414,6 +498,12 @@ static inline bool __COMPAT_has_generic_reenq(void)
return bpf_ksym_exists(scx_bpf_dsq_reenq___compat);
}
/* v7.4: Add scx_bpf_dsq_reenq_task(). */
static inline bool __COMPAT_has_dsq_reenq_task(void)
{
return bpf_ksym_exists(scx_bpf_dsq_reenq_task);
}
/*
* v6.19: The new void variant can be called from anywhere while the older v1
* variant can only be called from ops.cpu_release(). The double ___ prefixes on
tools/sched_ext/include/scx/compat.h
+23 -5
View File
@@ -214,6 +214,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
#define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP)
#define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE)
#define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED)
#define SCX_OPS_ENQ_BLOCKED SCX_OPS_FLAG(SCX_OPS_ENQ_BLOCKED)
#define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name)
@@ -274,23 +275,23 @@ static inline long scx_hotplug_seq(void)
* - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id
* - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us
*/
#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \
#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct, __opts) ({ \
struct __scx_name *__oskel; \
\
SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \
__ops_struct ".dump() missing, kernel too old?"); \
\
__oskel = __scx_name##__open(); \
__oskel = __scx_name##__open_opts(__opts); \
SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \
__oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
SCX_ENUM_INIT(__oskel); \
__oskel; \
})
#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \
#define SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \
struct __scx_name *__skel; \
\
__skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \
__skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops", __opts); \
if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \
!__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \
fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \
@@ -329,12 +330,29 @@ static inline long scx_hotplug_seq(void)
__skel; \
})
#define SCX_OPS_OPEN(__ops_name, __scx_name) \
SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, 0)
/*
* Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates
* every op the load-time fix-ups above handle, so none of them apply.
*
* A cid-form scheduler registers through bpf_scx_reg_cid() rather than
* bpf_scx_reg(), so repoint common.bpf.h's prolog probe at it. Both the cid
* form and bpf_scx_reg_cid() appeared in v7.2, so a kernel that accepts this
* skeleton always has the symbol.
*/
#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \
struct __scx_name *__cskel; \
\
__cskel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts); \
bpf_program__set_attach_target(__cskel->progs.scx_lib_init_probe, 0, \
"bpf_scx_reg_cid"); \
__cskel; \
})
#define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \
__SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid")
SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0)
/*
* Associate non-struct_ops BPF programs with the scheduler's struct_ops map so
tools/sched_ext/include/scx/const-defs.h
+20
View File
@@ -0,0 +1,20 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 Tejun Heo <tj@kernel.org> */
#pragma once
/*
* Constants shared across the BPF headers, the lib and the schedulers.
* Freestanding so that scheduler interface headers consumed by bindgen can
* include it. Rust code cannot, so rust/scx_arena/scx_arena/src/lib.rs mirrors
* the values. Keep them in sync.
*/
enum scx_const_defs {
/* mirrors the kernel's per-arch L1_CACHE_SHIFT */
#if defined(__TARGET_ARCH_s390) || defined(__s390x__)
SCX_CACHELINE_SIZE = 256,
#elif defined(__TARGET_ARCH_powerpc) || defined(__powerpc64__)
SCX_CACHELINE_SIZE = 128,
#else
SCX_CACHELINE_SIZE = 64,
#endif
};
tools/sched_ext/include/scx/enum_defs.autogen.h
+10
View File
@@ -85,10 +85,12 @@
#define HAVE_SCX_ENQ_HEAD
#define HAVE_SCX_ENQ_CPU_SELECTED
#define HAVE_SCX_ENQ_PREEMPT
#define HAVE_SCX_ENQ_PREEMPT_LAZY
#define HAVE_SCX_ENQ_IMMED
#define HAVE_SCX_ENQ_RESCUE
#define HAVE_SCX_ENQ_REENQ
#define HAVE_SCX_ENQ_LAST
#define HAVE_SCX_ENQ_BLOCKED
#define HAVE___SCX_ENQ_INTERNAL_MASK
#define HAVE_SCX_ENQ_CLEAR_OPSS
#define HAVE_SCX_ENQ_DSQ_PRIQ
@@ -105,6 +107,7 @@
#define HAVE_SCX_TASK_SUB_INIT
#define HAVE_SCX_TASK_IMMED
#define HAVE_SCX_TASK_PROTECTED
#define HAVE_SCX_TASK_RUN_TRACKED
#define HAVE_SCX_TASK_STATE_SHIFT
#define HAVE_SCX_TASK_STATE_BITS
#define HAVE_SCX_TASK_STATE_MASK
@@ -122,6 +125,7 @@
#define HAVE_SCX_TASK_REENQ_IMMED
#define HAVE_SCX_TASK_REENQ_PREEMPTED
#define HAVE_SCX_TASK_REENQ_CAP
#define HAVE_SCX_TASK_REENQ_PROXY
#define HAVE_SCX_TASK_CURSOR
#define HAVE_SCX_ECODE_RSN_HOTPLUG
#define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE
@@ -149,6 +153,8 @@
#define HAVE_SCX_KICK_IDLE
#define HAVE_SCX_KICK_PREEMPT
#define HAVE_SCX_KICK_WAIT
#define HAVE_SCX_KICK_PREEMPT_LAZY
#define HAVE_SCX_KICK_ALL_FLAGS
#define HAVE_SCX_OPI_BEGIN
#define HAVE_SCX_OPI_NORMAL_BEGIN
#define HAVE_SCX_OPI_NORMAL_END
@@ -164,6 +170,8 @@
#define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE
#define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED
#define HAVE_SCX_OPS_TID_TO_TASK
#define HAVE_SCX_OPS_LAZY_RESCHED
#define HAVE_SCX_OPS_ENQ_BLOCKED
#define HAVE_SCX_OPS_ALL_FLAGS
#define HAVE___SCX_OPS_INTERNAL_MASK
#define HAVE_SCX_OPS_HAS_CPU_PREEMPT
@@ -191,6 +199,8 @@
#define HAVE_SCX_RQ_BAL_CB_PENDING
#define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY
#define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY
#define HAVE_SCX_RQ_PROXY_RETRY
#define HAVE_SCX_RQ_PROXY_TICK
#define HAVE_SCX_RQ_IN_WAKEUP
#define HAVE_SCX_RQ_IN_DISPATCH
#define HAVE_SCX_SCHED_PCPU_BYPASSING
tools/sched_ext/include/scx/enums.autogen.bpf.h
+18 -3
View File
@@ -1,9 +1,9 @@
/*
* WARNING: This file is autogenerated from scripts/gen_enums.py. If you would
* like to access an enum that is currently missing, add it to the script
* and run it from the root directory to update this file.
* Definitions for SCX constants. Keep this table in sync
* with rust/scx_utils/src/enums.rs.
*/
const volatile u64 __SCX_OPS_NAME_LEN __weak;
#define SCX_OPS_NAME_LEN __SCX_OPS_NAME_LEN
@@ -22,6 +22,9 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak;
const volatile u64 __SCX_RQ_BAL_PENDING __weak;
#define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING
const volatile u64 __SCX_RQ_BAL_KEEP __weak;
#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP
const volatile u64 __SCX_RQ_BYPASSING __weak;
#define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING
@@ -112,6 +115,12 @@ const volatile u64 __SCX_KICK_PREEMPT __weak;
const volatile u64 __SCX_KICK_WAIT __weak;
#define SCX_KICK_WAIT __SCX_KICK_WAIT
const volatile u64 __SCX_KICK_PREEMPT_LAZY __weak;
#define SCX_KICK_PREEMPT_LAZY __SCX_KICK_PREEMPT_LAZY
const volatile u64 __SCX_KICK_ALL_FLAGS __weak;
#define SCX_KICK_ALL_FLAGS __SCX_KICK_ALL_FLAGS
const volatile u64 __SCX_ENQ_WAKEUP __weak;
#define SCX_ENQ_WAKEUP __SCX_ENQ_WAKEUP
@@ -121,12 +130,18 @@ const volatile u64 __SCX_ENQ_HEAD __weak;
const volatile u64 __SCX_ENQ_PREEMPT __weak;
#define SCX_ENQ_PREEMPT __SCX_ENQ_PREEMPT
const volatile u64 __SCX_ENQ_PREEMPT_LAZY __weak;
#define SCX_ENQ_PREEMPT_LAZY __SCX_ENQ_PREEMPT_LAZY
const volatile u64 __SCX_ENQ_IMMED __weak;
#define SCX_ENQ_IMMED __SCX_ENQ_IMMED
const volatile u64 __SCX_ENQ_RESCUE __weak;
#define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE
const volatile u64 __SCX_ENQ_BLOCKED __weak;
#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED
const volatile u64 __SCX_ENQ_REENQ __weak;
#define SCX_ENQ_REENQ __SCX_ENQ_REENQ
tools/sched_ext/include/scx/enums.autogen.h
+14 -8
View File
@@ -1,7 +1,8 @@
#pragma once
/*
* WARNING: This file is autogenerated from scripts/gen_enums.py. If you would
* like to access an enum that is currently missing, add it to the script
* and run it from the root directory to update this file.
* BTF enum lookups for SCX constants. Keep this table in sync
* with rust/scx_utils/src/enums.rs.
*/
#define SCX_ENUM_INIT(skel) do { \
@@ -11,6 +12,7 @@
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \
SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \
@@ -32,20 +34,24 @@
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \
SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NONE); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_INIT); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_READY); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_ENABLED); \
SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NR_STATES); \
SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT_LAZY); \
SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_ALL_FLAGS); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT_LAZY); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \
tools/sched_ext/include/scx/enums_abi.autogen.h
+11 -1
View File
@@ -97,10 +97,12 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU },
{ "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU },
{ "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU },
{ "scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", 0x800000000LLU },
{ "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU },
{ "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU },
{ "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU },
{ "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU },
{ "scx_enq_flags", "SCX_ENQ_BLOCKED", 0x40000000000LLU },
{ "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU },
{ "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU },
{ "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU },
@@ -117,6 +119,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU },
{ "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU },
{ "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU },
{ "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU },
{ "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU },
{ "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU },
{ "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU },
@@ -134,6 +137,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU },
{ "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU },
{ "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU },
{ "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU },
{ "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU },
{ "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU },
{ "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU },
@@ -161,6 +165,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU },
{ "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU },
{ "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU },
{ "scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", 0x8LLU },
{ "scx_kick_flags", "SCX_KICK_ALL_FLAGS", 0xfLLU },
{ "scx_opi", "SCX_OPI_BEGIN", 0x0LLU },
{ "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU },
{ "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU },
@@ -176,7 +182,9 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU },
{ "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU },
{ "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU },
{ "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU },
{ "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU },
{ "scx_ops_flags", "SCX_OPS_ENQ_BLOCKED", 0x400LLU },
{ "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x7ffLLU },
{ "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU },
{ "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU },
{ "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU },
@@ -203,6 +211,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
{ "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU },
{ "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU },
{ "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU },
{ "scx_rq_flags", "SCX_RQ_PROXY_RETRY", 0x200LLU },
{ "scx_rq_flags", "SCX_RQ_PROXY_TICK", 0x400LLU },
{ "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU },
{ "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU },
{ "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU },
tools/sched_ext/include/scx/features.bpf.h
+29
View File
@@ -0,0 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Kernel features the loader probes before load and hands to the BPF side in
* one rodata word, so that a program can carry both the path a feature enables
* and its fallback while the verifier prunes the one the running kernel cannot
* take. scx_utils sets the word in the ops open macros. The bits are mirrored
* in rust/scx_utils/src/compat.rs.
*
* Included by scx/common.bpf.h; don't include directly.
*
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
#ifndef __SCX_FEATURES_BPF_H
#define __SCX_FEATURES_BPF_H
enum scx_lib_feature {
/* the JIT lowers fetching AND, OR and XOR on arena pointers */
SCX_LIB_FEAT_ARENA_FETCH_BITOPS = 1ULL << 0,
};
const volatile u64 scx_lib_features __weak;
static __always_inline bool scx_lib_has(u64 feat)
{
return scx_lib_features & feat;
}
#endif /* __SCX_FEATURES_BPF_H */
tools/sched_ext/include/scx/user_exit_info.h
+1
View File
@@ -52,6 +52,7 @@
enum scx_exit_code {
/* Reasons */
SCX_ECODE_RSN_HOTPLUG = 1LLU << 32,
SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32,
/* Actions */
SCX_ECODE_ACT_RESTART = 1LLU << 48,
tools/sched_ext/scx_flatcg.bpf.c
+1 -1
View File
@@ -144,7 +144,7 @@ static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
cgc_a = container_of(a, struct cgv_node, rb_node);
cgc_b = container_of(b, struct cgv_node, rb_node);
return cgc_a->cvtime < cgc_b->cvtime;
return time_before(cgc_a->cvtime, cgc_b->cvtime);
}
static struct fcg_cpu_ctx *find_cpu_ctx(void)
tools/sched_ext/scx_qmap.bpf.c
+230 -39
View File
@@ -369,6 +369,22 @@ static u64 needs_immed(s32 cid)
return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0;
}
static void dispatch_to_rescue(struct task_struct *p, task_ctx_t *taskc,
u64 enq_flags)
{
u32 cid = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (cid >= scx_bpf_nr_cids()) {
scx_bpf_error("task %d has no allowed cid", p->pid);
return;
}
taskc->force_local = false;
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns,
enq_flags | SCX_ENQ_RESCUE);
}
/* first cid this node does NOT hold for fault injection, -1 if none */
static s32 first_unavail_cid(void)
{
@@ -442,6 +458,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
*/
taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++;
if (enq_flags & SCX_ENQ_BLOCKED)
__sync_fetch_and_add(&qa.nr_enq_blocked, 1);
/*
* A task of ours that can run on none of our self cids - the parent
* didn't grant them or we delegated them to children - would starve in
@@ -451,18 +470,64 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
* If we hold ENQ on that cid it runs. Otherwise the kernel diverts the
* task to its rescue path. IMMED would turn the insert into a legal
* placement on a time-shared cid and the kernel would bounce it back
* here instead of rescuing it.
* here instead of rescuing it. Do this before the blocked-donor fast
* paths, which also require an eligible self cid to make progress.
*/
if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
dispatch_to_rescue(p, taskc, enq_flags);
return;
}
if (c >= 0 && c < scx_bpf_nr_cids()) {
taskc->force_local = false;
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns,
enq_flags | SCX_ENQ_RESCUE);
/*
* SCX_OPS_ALWAYS_ENQ_IMMED makes the local insertion below implicitly
* carry SCX_ENQ_IMMED. If the CPU can't run the blocked donor immediately,
* the core returns it through ops.enqueue() with SCX_ENQ_REENQ. Inserting
* it into the same local DSQ would repeat the IMMED handback until the
* scheduler is ejected. Move reenqueued blocked donors to the shared DSQ,
* which doesn't carry SCX_ENQ_IMMED, so another CPU can consume them.
*/
if ((enq_flags & (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) ==
(SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) {
taskc->force_local = false;
scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags);
cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
&qa.idle_cids.mask,
&qa.self_cids.mask, 0);
if (cid < scx_bpf_nr_cids())
scx_bpf_kick_cid(cid, SCX_KICK_IDLE);
return;
}
/*
* Insert a blocked mutex donor at the head of an eligible local DSQ with
* a fresh slice and %SCX_ENQ_PREEMPT, requesting an immediate reschedule.
* The test above guarantees that cpus_allowed intersects self_cids, but
* the donor's current cid may have been delegated to a child. Search the
* intersection starting at the current cid, preserving it when qmap still
* holds it and wrapping to another eligible self cid otherwise.
*
* A self cid may be held exclusively with SCX_CAP_ENQ or time-shared with
* only SCX_CAP_ENQ_IMMED. Add needs_immed() so either kind can accept the
* local insertion instead of rejecting and reenqueuing the donor for a
* capability miss. Once selected, the core proxy-exec path can run the
* mutex owner using the donor's scheduling context.
*
* This policy is intentionally unfair and can strongly prioritize tasks
* using contended mutexes; scx_qmap is a demonstration scheduler and
* this behavior makes proxy-exec support easy to observe.
*/
if (enq_flags & SCX_ENQ_BLOCKED) {
cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
&qa.self_cids.mask,
scx_bpf_task_cid(p));
if (cid >= scx_bpf_nr_cids()) {
/* self_cids may have changed since the intersection test */
dispatch_to_rescue(p, taskc, enq_flags);
return;
}
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns,
enq_flags | needs_immed(cid) | SCX_ENQ_PREEMPT);
return;
}
/*
@@ -586,10 +651,35 @@ static void update_core_sched_head_seq(struct task_struct *p)
qa.core_sched_head_seqs[idx] = taskc->core_sched_seq;
}
/*
* Send @p, stranded on the DSQ @it iterates, to the kernel rescue path. To
* exercise both kfuncs, move tasks with an even pid there with
* scx_bpf_dsq_move() and re-enqueue the others with scx_bpf_dsq_reenq_task(),
* which takes them through qmap_enqueue()'s rescue. The choice is per task so
* that a marked task isn't moved by another scan before the re-enqueue reaches
* it.
*/
static __always_inline void rescue_stranded(struct bpf_iter_scx_dsq *it,
struct task_struct *p, task_ctx_t *taskc)
{
s32 c;
if (p->pid & 1) {
scx_bpf_dsq_reenq_task(it, p);
return;
}
c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < scx_bpf_nr_cids()) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(it, p, SCX_DSQ_LOCAL_ON | c, SCX_ENQ_RESCUE);
}
}
/*
* One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A
* task whose cids were lost while it was queued in the fifos would strand on
* SHARED_DSQ, which is consumed only on self cids it can't run on - move it to
* SHARED_DSQ, which is consumed only on self cids it can't run on - send it to
* the kernel rescue path. One whose cids were lost after the highpri cull is
* likewise rescued out of HIGHPRI_DSQ below.
*
@@ -598,9 +688,9 @@ static void update_core_sched_head_seq(struct task_struct *p)
* consuming them first. This makes minor difference only when dsp_batch is
* larger than 1.
*
* scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and
* non-rq-lock holding BPF programs. As demonstration, this function is called
* from qmap_dispatch() and monitor_timerfn().
* scx_bpf_dsq_move[_vtime]() and scx_bpf_dsq_reenq_task() are allowed both from
* ops.dispatch() and non-rq-lock holding BPF programs. As demonstration, this
* function is called from qmap_dispatch() and monitor_timerfn().
*/
static bool scan_shared_dsq(bool from_timer)
{
@@ -612,19 +702,13 @@ static bool scan_shared_dsq(bool from_timer)
bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) {
static u64 highpri_seq;
task_ctx_t *taskc;
s32 c;
if (!(taskc = lookup_task_ctx(p)))
return false;
/* stranded? rescue - it can't be dispatched here either way */
if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < scx_bpf_nr_cids()) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
SCX_ENQ_RESCUE);
}
rescue_stranded(BPF_FOR_EACH_ITER, p, taskc);
continue;
}
@@ -658,19 +742,12 @@ static bool scan_shared_dsq(bool from_timer)
&qa.usable_cids.mask,
this_cid + 1);
if (cid >= nr_cids) {
s32 c;
/* self cids lack caps in effect yet, leave it queued */
if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask))
continue;
/* stranded after the cull - rescue it from here */
c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < nr_cids) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
SCX_ENQ_RESCUE);
}
rescue_stranded(BPF_FOR_EACH_ITER, p, taskc);
continue;
}
@@ -1408,7 +1485,8 @@ __noinline void compute_partition(void)
/* find out the cids we hold */
scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask);
scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask);
cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */
/* held only as ENQ_IMMED */
cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask);
qa.part.nr_shared = 0;
qa.part.nr_rr = 0;
@@ -1563,8 +1641,7 @@ static __noinline void account_alloc(void)
*/
static void refresh_usable(void)
{
cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask);
cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask);
cmask_and(&qa.usable_scratch.mask, &qa.self_cids.mask, &qa.avail_cids.mask);
cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask);
}
@@ -1643,10 +1720,8 @@ __noinline void apply_partition(void)
if (!cgid)
continue;
cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask);
cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask);
cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask);
cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask);
cmask_andnot(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask, &ssc->granted_cids.mask);
cmask_andnot(&qa.to_grant_cids.mask, &ssc->granted_cids.mask, &ssc->prev_granted.mask);
scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF,
&qa.prev_rr_cids.mask);
@@ -1719,13 +1794,90 @@ static void redistribute(void)
}
/*
* Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading
* it for the stats display. Skipping when the partition guard is held is
* fine - alloc_ts is untouched, so the elapsed time is charged next time.
* Owner id for a @sched value of ops.sub_cid_sched_updated(). A child is
* attached before its first task runs and its tasks are re-homed before it
* detaches, so a child's cgroup id always has its slot.
*/
static s32 cid_sched_owner(u64 sched)
{
s32 i;
if (sched == SCX_CID_SCHED_NONE)
return CID_NONE;
if (sched == SCX_CID_SCHED_SELF)
return CID_SELF;
bpf_for(i, 0, MAX_SUB_SCHEDS)
if (qa.sub_sched_ctxs[i].cgroup_id == sched)
return i;
return CID_NONE;
}
/* used_ns[] is summed from every cpu, hence the atomic adds */
static void cid_sched_charge(s32 cid, u64 now)
{
s32 owner = qa.cid_sched[cid];
u64 delta = now - qa.cid_sched_since[cid];
if (owner >= 0 && owner < MAX_SUB_SCHEDS)
__sync_fetch_and_add(&qa.used_ns[owner], delta);
else if (owner == CID_SELF)
__sync_fetch_and_add(&qa.self_used_ns, delta);
qa.cid_sched_since[cid] = now;
}
void BPF_STRUCT_OPS(qmap_sub_cid_sched_updated, s32 cid, u64 sched)
{
if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS)
return;
cid_sched_charge(cid, bpf_ktime_get_ns());
qa.cid_sched[cid] = cid_sched_owner(sched);
}
/*
* Snapshot the used time for the stats display: the closed intervals plus the
* ones still open. The reads race the notifications on other cpus, so an
* interval closing in between can be missing from one snapshot or counted in
* two. The next snapshot evens it out and the display floors a negative
* difference at zero.
*/
static void snapshot_used(void)
{
u64 now = bpf_ktime_get_ns();
s32 nr_cids = qa.nr_cids;
s32 cid, i;
if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS)
return;
bpf_for(i, 0, MAX_SUB_SCHEDS)
qa.used_snap_ns[i] = qa.used_ns[i];
qa.self_used_snap_ns = qa.self_used_ns;
bpf_for(cid, 0, nr_cids) {
s32 owner = qa.cid_sched[cid];
u64 since = qa.cid_sched_since[cid];
/* restarted after @now by a notification on another cpu */
if (since > now)
continue;
if (owner >= 0 && owner < MAX_SUB_SCHEDS)
qa.used_snap_ns[owner] += now - since;
else if (owner == CID_SELF)
qa.self_used_snap_ns += now - since;
}
}
/*
* Userspace pokes this (PROG_RUN) to bring alloc_ns[] and the used snapshot
* current before reading them for the stats display. Skipping the alloc part
* when the partition guard is held is fine - alloc_ts is untouched, so the
* elapsed time is charged next time.
*/
SEC("syscall")
int flush_alloc(void *ctx)
{
snapshot_used();
if (part_try_start()) {
account_alloc();
part_end();
@@ -1877,6 +2029,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
/* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */
qa.nr_cids = nr_cids;
bpf_for(i, 0, nr_cids)
qa.cid_sched[i] = CID_NONE;
/* cmasks are embedded in qa, so they only need initializing */
cmask_init(&qa.idle_cids.mask, 0, nr_cids);
cmask_init(&qa.rr_cids.mask, 0, nr_cids);
@@ -1892,7 +2047,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask);
scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask);
cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask);
cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask);
bpf_for(i, 0, MAX_SUB_SCHEDS) {
cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids);
@@ -2024,12 +2179,36 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args)
void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args)
{
s32 i;
u64 cgid = args->ops->sub_cgroup_id;
s32 nr_cids = qa.nr_cids;
s32 pos = qa.part.rr_pos;
u64 holder_cgid;
bool reset_pool;
s32 i, cid;
if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) {
scx_bpf_error("-ERANGE");
return;
}
/*
* The child's caps are gone without a report, see ops.sub_detach().
* cpuperf targets persist until the next write. Reset the child's excl
* cids, and the pool unless another child holds it: that child's revoke
* report or its own detach resets it then.
*/
holder_cgid = qa.part.rr_slots[pos];
reset_pool = !holder_cgid || holder_cgid == cgid;
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id)
if (qa.sub_sched_ctxs[i].cgroup_id != cgid)
continue;
bpf_arena_for(cid, 0, nr_cids)
if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) ||
(reset_pool && cmask_test(cid, &qa.rr_cids.mask)))
scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE);
qa.sub_sched_ctxs[i].cgroup_id = 0;
qa.sub_sched_ctxs[i].weight = 100;
cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids);
@@ -2065,6 +2244,16 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after)
execute_partition();
}
void BPF_STRUCT_OPS(qmap_sub_child_ecaps_updated, u64 cgroup_id, s32 cid, u64 before,
u64 after)
{
__sync_fetch_and_add(&qa.nr_child_ecaps, 1);
/* a child's last target must not outlive its PERF cap */
if ((before & ~after) & SCX_CAP_PERF)
scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE);
}
SCX_OPS_CID_DEFINE(qmap_ops,
.flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK,
.select_cid = (void *)qmap_select_cid,
@@ -2088,6 +2277,8 @@ SCX_OPS_CID_DEFINE(qmap_ops,
.sub_detach = (void *)qmap_sub_detach,
.sub_caps_updated = (void *)qmap_sub_caps_updated,
.sub_ecaps_updated = (void *)qmap_sub_ecaps_updated,
.sub_cid_sched_updated = (void *)qmap_sub_cid_sched_updated,
.sub_child_ecaps_updated = (void *)qmap_sub_child_ecaps_updated,
.init_cids = (void *)qmap_init_cids,
.init = (void *)qmap_init,
.exit = (void *)qmap_exit,
tools/sched_ext/scx_qmap.c
+49 -11
View File
@@ -46,7 +46,7 @@ const char help_fmt[] =
"See the top-of-file comment in .bpf.c for the design.\n"
"\n"
"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n"
" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I] [-X]\n"
" [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n"
"\n"
" -s SLICE_US Override slice duration\n"
@@ -65,6 +65,7 @@ const char help_fmt[] =
" -S Suppress qmap-specific debug dump\n"
" -p Switch only tasks on SCHED_EXT policy instead of all\n"
" -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n"
" -X Turn on SCX_OPS_ENQ_BLOCKED\n"
" -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n"
" -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n"
" -i SEC Stats interval, seconds (default 5)\n"
@@ -104,11 +105,15 @@ struct hier_prev {
u64 alloc_ns[MAX_SUB_SCHEDS];
u64 self_alloc_ns;
u64 alloc_window_ns;
u64 used_snap_ns[MAX_SUB_SCHEDS];
u64 self_used_snap_ns;
u64 nr_dsps[MAX_SUB_SCHEDS];
u64 nr_reenq_cap;
u64 nr_reenq_immed;
u64 nr_enq_blocked;
u64 nr_inject_attempts;
u64 nr_rescue_dsp;
u64 nr_child_ecaps;
};
/* current wall-clock time as "HH:MM:SS" for the startup and interval headers */
@@ -157,10 +162,24 @@ static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_
strcpy(buf, "-");
}
/*
* Delta of a cumulative ns counter over the interval, as a fraction of the
* interval. The used snapshot can briefly run behind the previous one (see
* snapshot_used()), hence the floor.
*/
static double delta_ratio(u64 cur, u64 prev, double secs)
{
s64 delta = cur - prev;
return secs > 0 && delta > 0 ? delta / (secs * 1e9) : 0.0;
}
/* partition summary + one row per sched: weight, cpus, dispatch rate, cids */
static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid)
{
char ranges[128], who[16];
/* worst case, no ranges: a comma and up to six digits per cid */
static char ranges[SCX_QMAP_MAX_CPUS * 7];
char who[16];
const char *rr = "-";
double secs;
u32 i;
@@ -190,26 +209,39 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg
}
format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n",
printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu child_ecaps=+%llu\n",
(unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
(unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
(unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
(unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked),
(unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts),
(unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp));
(unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp),
(unsigned long long)(qa->nr_child_ecaps - prev->nr_child_ecaps));
prev->nr_reenq_cap = qa->nr_reenq_cap;
prev->nr_reenq_immed = qa->nr_reenq_immed;
prev->nr_enq_blocked = qa->nr_enq_blocked;
prev->nr_inject_attempts = qa->nr_inject_attempts;
prev->nr_rescue_dsp = qa->nr_rescue_dsp;
prev->nr_child_ecaps = qa->nr_child_ecaps;
printf("hier : %-4s %10s %4s %6s %8s %s\n",
"", "cgroup", "w", "alloc", "disp/s", "cids");
/*
* alloc is the cid-time the partition handed each participant, and used
* is the cid-time its tasks actually ran, per
* ops.sub_cid_sched_updated(). Both are in cpus over the window.
*/
printf("hier : %-4s %10s %4s %6s %6s %8s %s\n",
"", "cgroup", "w", "alloc", "used", "disp/s", "cids");
format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges));
printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self",
printf("hier : %-4s %10llu %4u %6.2f %6.2f %8s %s\n", "self",
(unsigned long long)own_cgid, 100,
secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0,
delta_ratio(qa->self_alloc_ns, prev->self_alloc_ns, secs),
delta_ratio(qa->self_used_snap_ns, prev->self_used_snap_ns, secs),
"-", ranges);
prev->self_alloc_ns = qa->self_alloc_ns;
/* used accrues without a window, so prev moves only with the window */
if (secs > 0)
prev->self_used_snap_ns = qa->self_used_snap_ns;
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i];
@@ -219,12 +251,15 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg
snprintf(who, sizeof(who), "sub%u", i);
format_cid_ranges(qa, i, ranges, sizeof(ranges));
printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who,
printf("hier : %-4s %10llu %4u %6.2f %6.2f %8.1f %s\n", who,
(unsigned long long)sc->cgroup_id, sc->weight,
secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0,
delta_ratio(qa->alloc_ns[i], prev->alloc_ns[i], secs),
delta_ratio(qa->used_snap_ns[i], prev->used_snap_ns[i], secs),
secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0,
ranges);
prev->alloc_ns[i] = qa->alloc_ns[i];
if (secs > 0)
prev->used_snap_ns[i] = qa->used_snap_ns[i];
prev->nr_dsps[i] = sc->nr_dsps;
}
}
@@ -263,7 +298,7 @@ restart:
skel->rodata->max_tasks = 16384;
while ((opt = getopt(argc, argv,
"s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) {
"s:e:t:T:l:b:N:PMHc:d:D:SpIXF:C:i:R:J:B:q:vh")) != -1) {
switch (opt) {
case 's':
skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
@@ -324,6 +359,9 @@ restart:
case 'I':
skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED;
break;
case 'X':
skel->struct_ops.qmap_ops->flags |= SCX_OPS_ENQ_BLOCKED;
break;
case 'F':
skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0);
break;
tools/sched_ext/scx_qmap.h
+13
View File
@@ -163,6 +163,17 @@ struct qmap_arena {
u64 alloc_ts; /* last accounting timestamp */
u64 alloc_window_ns; /* total accounted time, the alloc denominator */
/*
* The per-cid fields are written only by that cid's notifications and
* read by flush_alloc() for the snapshot userspace displays.
*/
s32 cid_sched[SCX_QMAP_MAX_CPUS]; /* per cid: owner id of the sched running there */
u64 cid_sched_since[SCX_QMAP_MAX_CPUS]; /* when that last changed */
u64 used_ns[MAX_SUB_SCHEDS]; /* per child slot, closed intervals */
u64 self_used_ns;
u64 used_snap_ns[MAX_SUB_SCHEDS]; /* used_ns[] plus the intervals open at the flush */
u64 self_used_snap_ns;
/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
struct qmap_cmask self_cids; /* cids this node runs its own tasks on */
struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */
@@ -181,8 +192,10 @@ struct qmap_arena {
/* bpf -> userspace: stats */
u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */
u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */
u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */
u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */
u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */
u64 nr_child_ecaps; /* ops.sub_child_ecaps_updated() deliveries */
u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */
};
tools/testing/selftests/sched_ext/.gitignore
+4
View File
@@ -4,3 +4,7 @@
!Makefile
!.gitignore
!config
!test_modules/
!test_modules/scx_enq_blocked_test.c
!test_modules/Makefile
test_modules/*.mod.c
tools/testing/selftests/sched_ext/Makefile
+9 -4
View File
@@ -5,10 +5,12 @@ include ../../../scripts/Makefile.arch
include ../../../scripts/Makefile.include
TEST_GEN_PROGS := runner
TEST_GEN_MODS_DIR := test_modules
# override lib.mk's default rules
OVERRIDE_TARGETS := 1
include ../lib.mk
include ../cgroup/lib/libcgroup.mk
CURDIR := $(abspath .)
REPOROOT := $(abspath ../../../..)
@@ -53,7 +55,7 @@ ifneq ($(wildcard $(GENHDR)),)
GENFLAGS := -DHAVE_GENHDR
endif
CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \
CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \
-I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
-I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include -I$(SCXTOOLSINCDIR)
@@ -62,7 +64,7 @@ ifneq ($(LLVM),)
CFLAGS += -Wno-unused-command-line-argument
endif
LDFLAGS = -lelf -lz -lpthread -lzstd
LDFLAGS += -lelf -lz -lpthread -lzstd
IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
@@ -153,7 +155,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
override define CLEAN
rm -rf $(OUTPUT_DIR)
rm -f $(TEST_GEN_PROGS)
rm -f $(TEST_GEN_PROGS) $(EXTRA_CLEAN)
endef
# Every testcase takes all of the BPF progs are dependencies by default. This
@@ -162,10 +164,12 @@ endef
all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog)))
auto-test-targets := \
cgroup_nr_cpus \
create_dsq \
dequeue \
dequeue_iter \
dequeue_remote \
enq_blocked \
enq_last_no_enq_fails \
ddsp_bogus_dsq_fail \
ddsp_vtimelocal_fail \
@@ -175,6 +179,7 @@ auto-test-targets := \
exit \
hotplug \
init_enable_count \
kick \
maximal \
maybe_null \
minimal \
@@ -214,7 +219,7 @@ $(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bp
$(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR)
$(CC) $(CFLAGS) -c $< -o $@
$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets)
$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(LIBCGROUP_O) $(testcase-targets)
@echo "$(testcase-targets)"
$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
tools/testing/selftests/sched_ext/allowed_cpus.bpf.c
+21 -3
View File
@@ -147,23 +147,41 @@ void BPF_STRUCT_OPS(allowed_cpus_exit, struct scx_exit_info *ei)
}
struct task_cpu_arg {
pid_t pid;
u64 pid;
s64 custom_cpu;
};
SEC("syscall")
int select_cpu_from_user(struct task_cpu_arg *input)
{
struct task_struct *p;
int cpu;
struct bpf_cpumask *mask;
s32 cpu;
p = bpf_task_from_pid(input->pid);
if (!p)
return -EINVAL;
mask = bpf_cpumask_create();
if (!mask) {
bpf_task_release(p);
return -ENOMEM;
}
/* A negative custom_cpu leaves the custom mask empty. */
if (input->custom_cpu >= 0)
bpf_cpumask_set_cpu(input->custom_cpu, mask);
bpf_rcu_read_lock();
cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, p->cpus_ptr, 0);
cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0,
cast_mask(mask), 0);
if (cpu >= 0 &&
(!bpf_cpumask_test_cpu(cpu, cast_mask(mask)) ||
!bpf_cpumask_test_cpu(cpu, &p->cpus_mask)))
cpu = -ERANGE;
bpf_rcu_read_unlock();
bpf_cpumask_release(mask);
bpf_task_release(p);
return cpu;
tools/testing/selftests/sched_ext/allowed_cpus.c
+111 -12
View File
@@ -2,7 +2,10 @@
/*
* Copyright (c) 2025 Andrea Righi <arighi@nvidia.com>
*/
#define _GNU_SOURCE
#include <bpf/bpf.h>
#include <limits.h>
#include <sched.h>
#include <scx/common.h>
#include <sys/wait.h>
#include <unistd.h>
@@ -23,17 +26,19 @@ static enum scx_test_status setup(void **ctx)
return SCX_TEST_PASS;
}
static int test_select_cpu_from_user(const struct allowed_cpus *skel)
static int test_select_cpu_from_user(const struct allowed_cpus *skel,
const char *name, int custom_cpu,
bool expect_busy)
{
int fd, ret;
__u64 args[1];
__s32 cpu;
__u64 args[] = { getpid(), (__u64)(__s64)custom_cpu };
LIBBPF_OPTS(bpf_test_run_opts, attr,
.ctx_in = args,
.ctx_size_in = sizeof(args),
);
args[0] = getpid();
fd = bpf_program__fd(skel->progs.select_cpu_from_user);
if (fd < 0)
return fd;
@@ -42,29 +47,123 @@ static int test_select_cpu_from_user(const struct allowed_cpus *skel)
if (ret < 0)
return ret;
fprintf(stderr, "%s: CPU %d\n", __func__, attr.retval);
/* test_run returns the signed BPF result through an unsigned field. */
cpu = (__s32)attr.retval;
if ((expect_busy && cpu != -EBUSY) ||
(!expect_busy && cpu != -EBUSY && cpu != custom_cpu)) {
SCX_ERR("%s: unexpected CPU selection result %d", name, cpu);
return -EINVAL;
}
return 0;
}
/* Grow until the mask covers the kernel's CPU range, including offline CPUs. */
static int alloc_affinity(cpu_set_t **mask, size_t *size)
{
int nr_cpus = CPU_SETSIZE;
cpu_set_t *cpus;
int err;
for (;;) {
*size = CPU_ALLOC_SIZE(nr_cpus);
cpus = CPU_ALLOC(nr_cpus);
if (!cpus)
return -ENOMEM;
CPU_ZERO_S(*size, cpus);
if (!sched_getaffinity(0, *size, cpus)) {
*mask = cpus;
return nr_cpus;
}
err = errno;
CPU_FREE(cpus);
if (err != EINVAL)
return -err;
if (nr_cpus > INT_MAX / 2)
return -EOVERFLOW;
nr_cpus *= 2;
}
}
static enum scx_test_status run(void *ctx)
{
struct allowed_cpus *skel = ctx;
struct bpf_link *link;
enum scx_test_status status = SCX_TEST_FAIL;
cpu_set_t *original = NULL, *pinned = NULL;
bool affinity_changed = false;
size_t size;
int first = -1, second = -1, cpu, nr_cpus;
struct bpf_link *link = NULL;
nr_cpus = alloc_affinity(&original, &size);
if (nr_cpus < 0) {
SCX_ERR("Failed to get affinity (%d)", -nr_cpus);
goto out;
}
pinned = CPU_ALLOC(nr_cpus);
if (!pinned) {
SCX_ERR("Failed to allocate affinity mask");
goto out;
}
for (cpu = 0; cpu < nr_cpus; cpu++) {
if (!CPU_ISSET_S(cpu, size, original))
continue;
if (first < 0) {
first = cpu;
} else {
second = cpu;
break;
}
}
if (first < 0) {
SCX_ERR("No CPU in affinity mask");
goto out;
}
link = bpf_map__attach_struct_ops(skel->maps.allowed_cpus_ops);
SCX_FAIL_IF(!link, "Failed to attach scheduler");
if (!link) {
SCX_ERR("Failed to attach scheduler");
goto out;
}
/* Pick an idle CPU from user-space */
SCX_FAIL_IF(test_select_cpu_from_user(skel), "Failed to pick idle CPU");
if (test_select_cpu_from_user(skel, "empty mask", -1, true))
goto out;
/* Just sleeping is fine, plenty of scheduling events happening */
/* A legal candidate may be busy; selection need not succeed. */
if (test_select_cpu_from_user(skel, "legal candidate", first, false))
goto out;
if (second >= 0) {
CPU_ZERO_S(size, pinned);
CPU_SET_S(first, size, pinned);
if (sched_setaffinity(0, size, pinned)) {
SCX_ERR("Failed to pin task (%d)", errno);
goto out;
}
affinity_changed = true;
if (test_select_cpu_from_user(skel, "disjoint masks", second, true))
goto out;
} else {
fprintf(stderr, "Skipping disjoint masks: need two allowed CPUs\n");
}
/* Just sleeping is fine, plenty of scheduling events happening. */
sleep(1);
if (UEI_EXITED(skel, uei)) {
SCX_ERR("Scheduler exited unexpectedly");
goto out;
}
status = SCX_TEST_PASS;
SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE));
out:
if (affinity_changed && sched_setaffinity(0, size, original)) {
SCX_ERR("Failed to restore affinity (%d)", errno);
status = SCX_TEST_FAIL;
}
bpf_link__destroy(link);
return SCX_TEST_PASS;
CPU_FREE(pinned);
CPU_FREE(original);
return status;
}
static void cleanup(void *ctx)
tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c
+60
View File
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Validate scx_bpf_cgroup_nr_cpus() from both BPF_PROG_TYPE_SYSCALL and
* struct_ops contexts.
*
* Copyright (c) 2026 NVIDIA Corporation.
*/
#include <scx/common.bpf.h>
char _license[] SEC("license") = "GPL";
UEI_DEFINE(uei);
/* input to cgroup_nr_cpus_read() */
u64 query_cgid;
/* output of cgroup_nr_cpus_read(), -1 if @query_cgid couldn't be resolved */
s64 query_nr_cpus = -1;
/* recorded by ops.cgroup_init() for @init_cgid */
u64 init_cgid;
s64 init_nr_cpus = -1;
SEC("syscall")
int cgroup_nr_cpus_read(void *ctx)
{
struct cgroup *cgrp;
query_nr_cpus = -1;
cgrp = bpf_cgroup_from_id(query_cgid);
if (!cgrp)
return -ENOENT;
query_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp);
bpf_cgroup_release(cgrp);
return 0;
}
s32 BPF_STRUCT_OPS(cgroup_nr_cpus_cgroup_init, struct cgroup *cgrp,
struct scx_cgroup_init_args *args)
{
if (cgrp->kn->id == init_cgid)
init_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp);
return 0;
}
void BPF_STRUCT_OPS(cgroup_nr_cpus_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops cgroup_nr_cpus_ops = {
.cgroup_init = (void *)cgroup_nr_cpus_cgroup_init,
.exit = (void *)cgroup_nr_cpus_exit,
.name = "cgroup_nr_cpus",
};
tools/testing/selftests/sched_ext/cgroup_nr_cpus.c
+419
View File
@@ -0,0 +1,419 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Verify that scx_bpf_cgroup_nr_cpus() reports the number of CPUs in a
* cgroup's effective cpuset, including inherited and updated cpusets.
*
* Copyright (c) 2026 NVIDIA Corporation.
*/
#define _GNU_SOURCE
#include <bpf/bpf.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <linux/limits.h>
#include <scx/common.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "cgroup_nr_cpus.bpf.skel.h"
#include "cgroup_util.h"
#include "scx_test.h"
/*
* Hierarchy under the cgroup2 root, all with the cpu controller enabled so
* that ops.cgroup_init() runs for each of them:
*
* parent cpuset enabled by the root, enables cpuset for its children
* parent/child owns a cpuset
* parent/child/leaf no cpuset of its own, inherits child's
*/
struct cgroup_nr_cpus_ctx {
struct cgroup_nr_cpus *skel;
struct bpf_link *link;
char root[PATH_MAX];
char parent[PATH_MAX];
char child[PATH_MAX];
char leaf[PATH_MAX];
bool parent_created;
bool child_created;
bool leaf_created;
};
static int join_path(char *dst, size_t dst_size, const char *parent, const char *name)
{
int ret;
ret = snprintf(dst, dst_size, "%s/%s", parent, name);
if (ret < 0 || (size_t)ret >= dst_size)
return -ENAMETOOLONG;
return 0;
}
static u64 cgroup_id(const char *path)
{
union {
u64 id;
unsigned char bytes[8];
} id = {};
struct file_handle *handle;
int mount_id, ret;
handle = calloc(1, sizeof(*handle) + sizeof(id));
if (!handle)
return 0;
handle->handle_bytes = sizeof(id);
ret = name_to_handle_at(AT_FDCWD, path, handle, &mount_id, 0);
if (!ret && handle->handle_bytes == sizeof(id))
memcpy(id.bytes, handle->f_handle, sizeof(id));
free(handle);
return ret ? 0 : id.id;
}
/*
* Parse a cpulist such as "0-3,8,10-11". Return the number of CPUs and the
* lowest and highest CPU in @first and @last, or -errno on failure.
*/
static int parse_cpulist(const char *cpulist, u32 *first, u32 *last)
{
const char *p = cpulist;
u32 lowest = UINT_MAX, highest = 0;
int count = 0;
while (*p && *p != '\n') {
unsigned long start, end_cpu;
char *end;
errno = 0;
start = strtoul(p, &end, 10);
if (errno || end == p || start > INT_MAX)
return -EINVAL;
end_cpu = start;
p = end;
if (*p == '-') {
end_cpu = strtoul(p + 1, &end, 10);
if (errno || end == p + 1 || end_cpu > INT_MAX || end_cpu < start)
return -EINVAL;
p = end;
}
if (end_cpu - start + 1 > (unsigned long)(INT_MAX - count))
return -EOVERFLOW;
if (start < lowest)
lowest = start;
if (end_cpu > highest)
highest = end_cpu;
count += end_cpu - start + 1;
if (*p == ',')
p++;
else if (*p && *p != '\n')
return -EINVAL;
}
if (first)
*first = lowest;
if (last)
*last = highest;
return count;
}
/*
* Number of CPUs in @cgroup's cpuset.cpus.effective, or -errno.
*
* A sparse cpulist can exceed a page on large systems. cg_read() does a single
* bounded read, so size the buffer for the worst case of NR_CPUS=8192 and
* reject a read that fills it rather than parsing a truncated list.
*/
static int effective_nr_cpus(const char *cgroup, u32 *first, u32 *last)
{
static char buf[65536];
if (cg_read(cgroup, "cpuset.cpus.effective", buf, sizeof(buf)))
return -EIO;
if (strlen(buf) >= sizeof(buf) - 1)
return -EOVERFLOW;
return parse_cpulist(buf, first, last);
}
/* Run the SYSCALL program to sample scx_bpf_cgroup_nr_cpus() for @path. */
static int kfunc_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, s64 *nr_cpus)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
u64 cgid;
int err;
cgid = cgroup_id(path);
if (!cgid) {
SCX_ERR("Failed to read cgroup ID of %s", path);
return -ENOENT;
}
ctx->skel->bss->query_cgid = cgid;
err = bpf_prog_test_run_opts(bpf_program__fd(ctx->skel->progs.cgroup_nr_cpus_read),
&topts);
if (err || topts.retval) {
SCX_ERR("BPF_PROG_RUN failed for %s (err=%d retval=%d)",
path, err, (int)topts.retval);
return err ?: -EIO;
}
*nr_cpus = ctx->skel->data->query_nr_cpus;
return 0;
}
static bool check_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected,
const char *what)
{
s64 nr_cpus;
if (kfunc_nr_cpus(ctx, path, &nr_cpus))
return false;
if (nr_cpus != expected) {
SCX_ERR("%s: expected %d CPUs, got %lld", what, expected,
(long long)nr_cpus);
return false;
}
return true;
}
/*
* Like check_nr_cpus() but tolerate a transient mismatch. A cpuset css being
* disabled stays attached to its cgroup until it's asynchronously offlined, and
* cpuset_num_cpus() keeps reporting its stale mask until then.
*/
static bool wait_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected,
const char *what)
{
s64 nr_cpus = -1;
int i;
for (i = 0; i < 1000; i++) {
if (kfunc_nr_cpus(ctx, path, &nr_cpus))
return false;
if (nr_cpus == expected)
return true;
usleep(1000);
}
SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, (long long)nr_cpus);
return false;
}
static bool controller_enabled(const char *cgroup, const char *file, const char *controller)
{
char buf[4096], *saveptr, *token;
if (cg_read(cgroup, file, buf, sizeof(buf)))
return false;
for (token = strtok_r(buf, "\n ", &saveptr); token;
token = strtok_r(NULL, "\n ", &saveptr))
if (!strcmp(token, controller))
return true;
return false;
}
static void cleanup_ctx(struct cgroup_nr_cpus_ctx *ctx)
{
bpf_link__destroy(ctx->link);
cgroup_nr_cpus__destroy(ctx->skel);
if (ctx->leaf_created)
cg_destroy(ctx->leaf);
if (ctx->child_created)
cg_destroy(ctx->child);
if (ctx->parent_created)
cg_destroy(ctx->parent);
}
/*
* Enable @controller in the root's subtree_control if needed. Like the cgroup
* selftests, leave it enabled afterwards: the root is shared, and another
* manager may start relying on the controller while the test runs.
*/
static enum scx_test_status enable_controller(const char *root, const char *controller)
{
char value[32];
if (controller_enabled(root, "cgroup.subtree_control", controller))
return SCX_TEST_PASS;
if (!controller_enabled(root, "cgroup.controllers", controller))
return SCX_TEST_SKIP;
snprintf(value, sizeof(value), "+%s", controller);
if (cg_write(root, "cgroup.subtree_control", value))
return SCX_TEST_SKIP;
return SCX_TEST_PASS;
}
static enum scx_test_status setup_cgroups(struct cgroup_nr_cpus_ctx *ctx)
{
enum scx_test_status status;
char name[64];
if (cg_find_unified_root(ctx->root, sizeof(ctx->root), NULL))
return SCX_TEST_SKIP;
status = enable_controller(ctx->root, "cpu");
if (status != SCX_TEST_PASS)
return status;
status = enable_controller(ctx->root, "cpuset");
if (status != SCX_TEST_PASS)
return status;
snprintf(name, sizeof(name), "scx_nr_cpus_%d", getpid());
if (join_path(ctx->parent, sizeof(ctx->parent), ctx->root, name) ||
join_path(ctx->child, sizeof(ctx->child), ctx->parent, "child") ||
join_path(ctx->leaf, sizeof(ctx->leaf), ctx->child, "leaf")) {
SCX_ERR("Cgroup path is too long");
return SCX_TEST_FAIL;
}
if (cg_create(ctx->parent)) {
SCX_ERR("Failed to create cgroup %s", ctx->parent);
return SCX_TEST_FAIL;
}
ctx->parent_created = true;
if (cg_write(ctx->parent, "cgroup.subtree_control", "+cpu +cpuset")) {
SCX_ERR("Failed to enable controllers in %s", ctx->parent);
return SCX_TEST_FAIL;
}
if (cg_create(ctx->child)) {
SCX_ERR("Failed to create cgroup %s", ctx->child);
return SCX_TEST_FAIL;
}
ctx->child_created = true;
if (cg_write(ctx->child, "cgroup.subtree_control", "+cpu")) {
SCX_ERR("Failed to enable cpu in %s", ctx->child);
return SCX_TEST_FAIL;
}
if (cg_create(ctx->leaf)) {
SCX_ERR("Failed to create cgroup %s", ctx->leaf);
return SCX_TEST_FAIL;
}
ctx->leaf_created = true;
return SCX_TEST_PASS;
}
static enum scx_test_status run(void *arg)
{
struct cgroup_nr_cpus_ctx ctx = {};
enum scx_test_status status;
char value[32];
u32 first, last;
int nr_root, nr_child;
(void)arg;
/*
* SCX_ENUM_INIT() exits the process if vmlinux BTF can't be loaded, so
* run it before creating any cgroups that would then be left behind.
*/
ctx.skel = cgroup_nr_cpus__open();
if (!ctx.skel) {
SCX_ERR("Failed to open skel");
return SCX_TEST_FAIL;
}
SCX_ENUM_INIT(ctx.skel);
status = setup_cgroups(&ctx);
if (status != SCX_TEST_PASS)
goto out;
status = SCX_TEST_FAIL;
nr_root = effective_nr_cpus(ctx.root, NULL, NULL);
nr_child = effective_nr_cpus(ctx.child, &first, &last);
if (nr_root < 0 || nr_child < 0) {
SCX_ERR("Failed to read effective cpusets");
goto out;
}
/* The effective cpuset can be empty, e.g. under a partition root. */
if (nr_child < 2) {
status = SCX_TEST_SKIP;
goto out;
}
ctx.skel->bss->init_cgid = cgroup_id(ctx.leaf);
if (!ctx.skel->bss->init_cgid) {
SCX_ERR("Failed to read cgroup ID of %s", ctx.leaf);
goto out;
}
if (cgroup_nr_cpus__load(ctx.skel)) {
SCX_ERR("Failed to load skel");
goto out;
}
/* The kfunc must be callable without a scheduler attached. */
if (!check_nr_cpus(&ctx, ctx.root, nr_root, "root") ||
!check_nr_cpus(&ctx, ctx.child, nr_child, "child") ||
!check_nr_cpus(&ctx, ctx.leaf, nr_child, "inherited leaf"))
goto out;
/* ops.cgroup_init() runs for existing cgroups when attaching. */
ctx.link = bpf_map__attach_struct_ops(ctx.skel->maps.cgroup_nr_cpus_ops);
if (!ctx.link) {
SCX_ERR("Failed to attach scheduler");
goto out;
}
if (ctx.skel->data->init_nr_cpus != nr_child) {
SCX_ERR("ops.cgroup_init(): expected %d CPUs, got %lld", nr_child,
(long long)ctx.skel->data->init_nr_cpus);
goto out;
}
/* Non-contiguous cpuset, observed by the owner and by the inheritor. */
if (nr_child > 2) {
snprintf(value, sizeof(value), "%u,%u", first, last);
if (cg_write(ctx.child, "cpuset.cpus", value)) {
SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child);
goto out;
}
if (!check_nr_cpus(&ctx, ctx.child, 2, "sparse child") ||
!check_nr_cpus(&ctx, ctx.leaf, 2, "sparse inherited leaf"))
goto out;
}
snprintf(value, sizeof(value), "%u", first);
if (cg_write(ctx.child, "cpuset.cpus", value)) {
SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child);
goto out;
}
if (!check_nr_cpus(&ctx, ctx.child, 1, "single-CPU child") ||
!check_nr_cpus(&ctx, ctx.leaf, 1, "single-CPU inherited leaf"))
goto out;
/*
* Disabling cpuset below @parent makes @child and @leaf inherit
* @parent's effective cpuset, which spans all of the root's CPUs.
*/
if (cg_write(ctx.parent, "cgroup.subtree_control", "-cpuset")) {
SCX_ERR("Failed to disable cpuset in %s", ctx.parent);
goto out;
}
nr_child = effective_nr_cpus(ctx.parent, NULL, NULL);
if (nr_child < 0) {
SCX_ERR("Failed to read effective cpuset of %s", ctx.parent);
goto out;
}
if (!wait_nr_cpus(&ctx, ctx.child, nr_child, "child after cpuset disable") ||
!wait_nr_cpus(&ctx, ctx.leaf, nr_child, "leaf after cpuset disable"))
goto out;
if (UEI_EXITED(ctx.skel, uei)) {
SCX_ERR("Scheduler exited unexpectedly");
goto out;
}
status = SCX_TEST_PASS;
out:
cleanup_ctx(&ctx);
return status;
}
struct scx_test cgroup_nr_cpus = {
.name = "cgroup_nr_cpus",
.description = "Verify scx_bpf_cgroup_nr_cpus() reports effective cpuset CPU counts",
.run = run,
};
REGISTER_SCX_TEST(&cgroup_nr_cpus)
tools/testing/selftests/sched_ext/config
+3
View File
@@ -1,8 +1,11 @@
CONFIG_SCHED_CLASS_EXT=y
CONFIG_CGROUPS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CPUSETS=y
CONFIG_EXT_GROUP_SCHED=y
CONFIG_BPF=y
CONFIG_BPF_SYSCALL=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_EXPERT=y
CONFIG_SCHED_PROXY_EXEC=y
tools/testing/selftests/sched_ext/create_dsq.bpf.c
+43
View File
@@ -10,6 +10,10 @@
char _license[] SEC("license") = "GPL";
u32 nr_lifecycle_tests;
UEI_DEFINE(uei);
void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p,
struct scx_exit_task_args *args)
{
@@ -43,16 +47,55 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init)
}
bpf_for(i, 0, 1024) {
err = scx_bpf_create_dsq(i, -1);
if (err != -EEXIST) {
scx_bpf_error("Duplicate DSQ %d creation returned %d", i, err);
return -EINVAL;
}
/* A rejected duplicate must leave the original DSQ accessible. */
err = scx_bpf_dsq_nr_queued(i);
if (err) {
scx_bpf_error("Original DSQ %d queue count is %d", i, err);
return -EINVAL;
}
scx_bpf_destroy_dsq(i);
err = scx_bpf_dsq_nr_queued(i);
if (err != -ENOENT) {
scx_bpf_error("Destroyed DSQ %d queue count is %d", i, err);
return -EINVAL;
}
err = scx_bpf_create_dsq(i, -1);
if (err) {
scx_bpf_error("Failed to recreate DSQ %d: %d", i, err);
return err;
}
err = scx_bpf_dsq_nr_queued(i);
if (err) {
scx_bpf_error("Recreated DSQ %d queue count is %d", i, err);
return -EINVAL;
}
scx_bpf_destroy_dsq(i);
nr_lifecycle_tests++;
}
return 0;
}
void BPF_STRUCT_OPS(create_dsq_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops create_dsq_ops = {
.init_task = (void *) create_dsq_init_task,
.exit_task = (void *) create_dsq_exit_task,
.init = (void *) create_dsq_init,
.exit = (void *) create_dsq_exit,
.name = "create_dsq",
};
tools/testing/selftests/sched_ext/create_dsq.c
+5 -1
View File
@@ -35,8 +35,12 @@ static enum scx_test_status run(void *ctx)
return SCX_TEST_FAIL;
}
SCX_ASSERT_ALIVE(skel, uei);
bpf_link__destroy(link);
SCX_EQ(skel->bss->nr_lifecycle_tests, 1024);
return SCX_TEST_PASS;
}
@@ -49,7 +53,7 @@ static void cleanup(void *ctx)
struct scx_test create_dsq = {
.name = "create_dsq",
.description = "Create and destroy a dsq in a loop",
.description = "Create, reject duplicates, destroy and recreate DSQs",
.setup = setup,
.run = run,
.cleanup = cleanup,
tools/testing/selftests/sched_ext/cyclic_kick_wait.c
+1 -1
View File
@@ -152,7 +152,7 @@ static enum scx_test_status run(void *ctx)
sleep(5);
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
if (UEI_EXITED(skel, uei)) {
SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
(unsigned long long)skel->data->uei.kind,
(long long)skel->data->uei.exit_code);
tools/testing/selftests/sched_ext/dequeue_remote.c
+1
View File
@@ -43,6 +43,7 @@ static void worker_fn(int id)
for (j = 0; j < 20000 + id * 1000; j++)
sum += j;
asm volatile("" : : "r"(sum));
if (id & 1)
usleep(100);
else
tools/testing/selftests/sched_ext/enq_blocked.bpf.c
+116
View File
@@ -0,0 +1,116 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*
* Verify that SCX_OPS_ENQ_BLOCKED passes blocked proxy donors through
* ops.enqueue() and record whether callbacks occur on the donor or owner CPU.
*/
#include <scx/common.bpf.h>
#define SHARED_DSQ 0
char _license[] SEC("license") = "GPL";
s32 donor_pid;
s32 donor_cpu = -1;
s32 owner_cpu = -1;
u64 nr_blocked_enqueues;
u64 nr_blocked_enqueues_donor_cpu;
u64 nr_blocked_enqueues_owner_cpu;
u64 nr_blocked_enqueues_other_cpu;
u64 nr_blocked_wakeups;
static u64 vtime_now;
UEI_DEFINE(uei);
s32 BPF_STRUCT_OPS(enq_blocked_select_cpu,
struct task_struct *p, s32 prev_cpu, u64 wake_flags)
{
return prev_cpu;
}
void BPF_STRUCT_OPS(enq_blocked_enqueue, struct task_struct *p, u64 enq_flags)
{
u64 vtime = p->scx.dsq_vtime;
if (enq_flags & SCX_ENQ_BLOCKED) {
int cpu = scx_bpf_task_cpu(p);
if (enq_flags & SCX_ENQ_WAKEUP)
__sync_fetch_and_add(&nr_blocked_wakeups, 1);
if (p->pid == donor_pid) {
__sync_fetch_and_add(&nr_blocked_enqueues, 1);
if (cpu == donor_cpu)
__sync_fetch_and_add(&nr_blocked_enqueues_donor_cpu, 1);
else if (cpu == owner_cpu)
__sync_fetch_and_add(&nr_blocked_enqueues_owner_cpu, 1);
else
__sync_fetch_and_add(&nr_blocked_enqueues_other_cpu, 1);
}
}
/* Limit the amount of budget an idling task can accumulate. */
if (time_before(vtime, vtime_now - SCX_SLICE_DFL))
vtime = vtime_now - SCX_SLICE_DFL;
scx_bpf_dsq_insert_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime,
enq_flags);
scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE);
}
void BPF_STRUCT_OPS(enq_blocked_dispatch, s32 cpu, struct task_struct *prev)
{
scx_bpf_dsq_move_to_local(SHARED_DSQ, 0);
}
void BPF_STRUCT_OPS(enq_blocked_running, struct task_struct *p)
{
if (time_before(vtime_now, p->scx.dsq_vtime))
vtime_now = p->scx.dsq_vtime;
}
void BPF_STRUCT_OPS(enq_blocked_stopping, struct task_struct *p, bool runnable)
{
u64 delta = scale_by_task_weight_inverse(p,
SCX_SLICE_DFL - p->scx.slice);
scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta);
}
void BPF_STRUCT_OPS(enq_blocked_enable, struct task_struct *p)
{
scx_bpf_task_set_dsq_vtime(p, vtime_now);
}
s32 BPF_STRUCT_OPS_SLEEPABLE(enq_blocked_init)
{
int ret;
ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
if (ret) {
scx_bpf_error("failed to create DSQ %d (%d)", SHARED_DSQ, ret);
return ret;
}
return 0;
}
void BPF_STRUCT_OPS(enq_blocked_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops enq_blocked_ops = {
.select_cpu = (void *)enq_blocked_select_cpu,
.enqueue = (void *)enq_blocked_enqueue,
.dispatch = (void *)enq_blocked_dispatch,
.running = (void *)enq_blocked_running,
.stopping = (void *)enq_blocked_stopping,
.enable = (void *)enq_blocked_enable,
.init = (void *)enq_blocked_init,
.exit = (void *)enq_blocked_exit,
.name = "enq_blocked",
};
tools/testing/selftests/sched_ext/enq_blocked.c
+917
View File
@@ -0,0 +1,917 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*
* Exercise a priority inversion with the owner and donor first pinned to the
* same CPU, then with each on a different CPU. A high-priority donor blocks on
* a mutex held by a low-priority owner while one medium-priority contender per
* available CPU keeps the system busy. A weighted-vruntime BPF scheduler runs
* both CPU placement configurations with SCX_OPS_ENQ_BLOCKED first disabled
* and then enabled. The test validates blocked-donor admission and reports the
* average mutex hold and wait times, plus their enabled-minus-disabled deltas,
* for each configuration. The timing data is informational.
*
* CONFIG_SCHED_PROXY_EXEC=y is required to exercise the proxy-execution paths.
*/
#define _GNU_SOURCE
#include <bpf/bpf.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <pthread.h>
#include <sched.h>
#include <scx/common.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>
#include "enq_blocked.bpf.skel.h"
#include "enq_blocked.h"
#include "scx_test.h"
#define MODULE_NAME "scx_enq_blocked_test"
#define MODULE_FILE "test_modules/" MODULE_NAME ".ko"
#define DEVICE_PATH "/dev/scx_enq_blocked"
#define WAIT_STEP_US 1000
#define WAIT_TIMEOUT_MS 2000
#define NR_WARMUP_TRIALS 1
#define NR_MEASURED_TRIALS 10
#define NR_TRIALS (NR_WARMUP_TRIALS + NR_MEASURED_TRIALS)
#define JOIN_TIMEOUT_MS ((NR_TRIALS + 1) * WAIT_TIMEOUT_MS)
#define OWNER_NICE 19
#define DONOR_NICE -20
#define CONTENDER_NICE 0
struct thread_ctx {
atomic_bool start_donor;
atomic_bool abort;
atomic_bool stop_contender;
atomic_bool measurement_ready;
atomic_int donor_pid;
atomic_int donor_completed;
int fd;
int donor_cpu;
int owner_cpu;
};
struct contender_ctx {
struct thread_ctx *thread_ctx;
atomic_int status;
int cpu;
};
struct run_result {
struct enq_blocked_stats stats;
u64 nr_blocked_enqueues;
u64 nr_blocked_enqueues_donor_cpu;
u64 nr_blocked_enqueues_owner_cpu;
u64 nr_blocked_enqueues_other_cpu;
u64 nr_blocked_wakeups;
};
static bool parse_bool(const char *value, bool *result)
{
if (!strcasecmp(value, "1") || !strcasecmp(value, "y") ||
!strcasecmp(value, "yes") || !strcasecmp(value, "on") ||
!strcasecmp(value, "true")) {
*result = true;
return true;
}
if (!strcasecmp(value, "0") || !strcasecmp(value, "n") ||
!strcasecmp(value, "no") || !strcasecmp(value, "off") ||
!strcasecmp(value, "false")) {
*result = false;
return true;
}
return false;
}
static bool cmdline_bool(const char *name, bool default_value)
{
char cmdline[4096], *newline, *saveptr = NULL, *token;
size_t name_len = strlen(name);
bool value = default_value;
FILE *file;
file = fopen("/proc/cmdline", "r");
if (!file)
return default_value;
if (!fgets(cmdline, sizeof(cmdline), file)) {
fclose(file);
return default_value;
}
fclose(file);
newline = strchr(cmdline, '\n');
if (newline)
*newline = '\0';
for (token = strtok_r(cmdline, " ", &saveptr); token;
token = strtok_r(NULL, " ", &saveptr)) {
bool parsed;
if (strncmp(token, name, name_len) || token[name_len] != '=')
continue;
if (parse_bool(token + name_len + 1, &parsed))
value = parsed;
}
return value;
}
static int module_path(char *path, size_t size)
{
ssize_t len;
char *slash;
len = readlink("/proc/self/exe", path, size - 1);
if (len < 0)
return -errno;
path[len] = '\0';
slash = strrchr(path, '/');
if (!slash)
return -EINVAL;
*slash = '\0';
if (snprintf(slash, size - (slash - path), "/%s", MODULE_FILE) >=
size - (slash - path))
return -ENAMETOOLONG;
return 0;
}
static int load_test_module(bool *loaded_here)
{
char path[PATH_MAX];
int fd, err;
err = module_path(path, sizeof(path));
if (err)
return err;
fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
return -errno;
if (syscall(SYS_finit_module, fd, "", 0)) {
err = errno;
close(fd);
if (err == EEXIST)
return 0;
return -err;
}
close(fd);
*loaded_here = true;
return 0;
}
static void unload_test_module(bool loaded_here)
{
if (loaded_here && syscall(SYS_delete_module, MODULE_NAME, O_NONBLOCK))
SCX_ERR("Failed to unload %s (%d)", MODULE_NAME, errno);
}
static int pin_to_cpu(int cpu)
{
cpu_set_t mask;
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
return sched_setaffinity(0, sizeof(mask), &mask) ? errno : 0;
}
static int select_test_cpus(bool cross_cpu, cpu_set_t *mask, int *donor_cpu,
int *owner_cpu)
{
int cpu, first = -1;
if (sched_getaffinity(0, sizeof(*mask), mask))
return -errno;
for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
if (!CPU_ISSET(cpu, mask))
continue;
if (first < 0) {
first = cpu;
if (!cross_cpu)
break;
} else {
*donor_cpu = first;
*owner_cpu = cpu;
return 0;
}
}
if (first < 0)
return -ENODEV;
if (cross_cpu)
return -EAGAIN;
*donor_cpu = first;
*owner_cpu = first;
return 0;
}
static int set_nice(int nice)
{
return setpriority(PRIO_PROCESS, 0, nice) ? errno : 0;
}
static bool wait_for_pid(atomic_int *pid)
{
int waited_ms;
for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) {
if (atomic_load_explicit(pid, memory_order_acquire) > 0)
return true;
usleep(WAIT_STEP_US);
}
return false;
}
static int wait_for_contenders(struct contender_ctx *contenders,
size_t nr_contenders)
{
size_t i, nr_ready;
int status, waited_ms;
for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) {
nr_ready = 0;
for (i = 0; i < nr_contenders; i++) {
status = atomic_load_explicit(&contenders[i].status,
memory_order_acquire);
if (status < 0)
return status;
if (status > 0)
nr_ready++;
}
if (nr_ready == nr_contenders)
return 1;
usleep(WAIT_STEP_US);
}
return -ETIMEDOUT;
}
static int wait_for_donor_state(struct thread_ctx *ctx, int expected)
{
int state, waited_ms;
for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) {
state = ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR_STATE);
if (state == expected)
return state;
if (state < 0 && errno != ENOENT)
return -errno;
usleep(WAIT_STEP_US);
}
return -ETIMEDOUT;
}
static bool wait_for_donor(struct thread_ctx *ctx, int trial)
{
int waited_ms;
for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) {
if (atomic_load_explicit(&ctx->donor_completed,
memory_order_acquire) >= trial)
return true;
if (atomic_load_explicit(&ctx->abort, memory_order_relaxed))
return false;
usleep(WAIT_STEP_US);
}
return false;
}
static bool wait_for_measurement(struct thread_ctx *ctx)
{
while (!atomic_load_explicit(&ctx->measurement_ready,
memory_order_acquire) &&
!atomic_load_explicit(&ctx->abort, memory_order_relaxed))
sched_yield();
return !atomic_load_explicit(&ctx->abort, memory_order_relaxed);
}
static void *contender_fn(void *arg)
{
struct contender_ctx *contender = arg;
struct thread_ctx *ctx = contender->thread_ctx;
int err;
err = pin_to_cpu(contender->cpu);
if (!err)
err = set_nice(CONTENDER_NICE);
atomic_store_explicit(&contender->status, err ? -err : 1,
memory_order_release);
if (err)
return (void *)(uintptr_t)err;
while (!atomic_load_explicit(&ctx->stop_contender,
memory_order_relaxed))
;
return NULL;
}
static void *owner_fn(void *arg)
{
struct thread_ctx *ctx = arg;
int err, i;
err = pin_to_cpu(ctx->owner_cpu);
if (err)
return (void *)(uintptr_t)err;
err = set_nice(OWNER_NICE);
if (err)
return (void *)(uintptr_t)err;
for (i = 0; i < NR_TRIALS; i++) {
if (ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_OWNER))
return (void *)(uintptr_t)errno;
if (!wait_for_donor(ctx, i + 1))
return (void *)(uintptr_t)ETIMEDOUT;
if (i + 1 == NR_WARMUP_TRIALS && !wait_for_measurement(ctx))
return NULL;
}
return NULL;
}
static int run_donor_trial(struct thread_ctx *ctx)
{
int waited_ms;
for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) {
if (!ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR))
return 0;
if (errno != EAGAIN)
return -errno;
usleep(WAIT_STEP_US);
}
return -ETIMEDOUT;
}
static void *donor_fn(void *arg)
{
struct thread_ctx *ctx = arg;
int err, i;
err = pin_to_cpu(ctx->donor_cpu);
if (err)
return (void *)(uintptr_t)err;
err = set_nice(DONOR_NICE);
if (err)
return (void *)(uintptr_t)err;
atomic_store_explicit(&ctx->donor_pid, syscall(SYS_gettid),
memory_order_release);
while (!atomic_load_explicit(&ctx->start_donor, memory_order_acquire) &&
!atomic_load_explicit(&ctx->abort, memory_order_relaxed))
sched_yield();
if (atomic_load_explicit(&ctx->abort, memory_order_relaxed))
return NULL;
for (i = 0; i < NR_TRIALS; i++) {
err = run_donor_trial(ctx);
if (err)
return (void *)(uintptr_t)-err;
atomic_store_explicit(&ctx->donor_completed, i + 1,
memory_order_release);
}
return NULL;
}
static void print_avg_time(const char *name, u64 total_ns, u64 samples)
{
u64 avg_ns = samples ? total_ns / samples : 0;
printf(" %s_avg_ns=%llu (%llu.%03llu ms, samples=%llu)\n", name,
(unsigned long long)avg_ns,
(unsigned long long)(avg_ns / 1000000),
(unsigned long long)((avg_ns / 1000) % 1000),
(unsigned long long)samples);
}
static void print_avg_delta(const char *name, u64 disabled_total,
u64 disabled_samples, u64 enabled_total,
u64 enabled_samples)
{
u64 disabled_avg, enabled_avg;
s64 delta_ns;
double delta_pct;
if (!disabled_samples || !enabled_samples)
return;
disabled_avg = disabled_total / disabled_samples;
enabled_avg = enabled_total / enabled_samples;
delta_ns = (s64)enabled_avg - (s64)disabled_avg;
delta_pct = disabled_avg ? 100.0 * delta_ns / disabled_avg : 0.0;
printf(" %s_delta_ns=%+lld (%+.2f%%)\n", name,
(long long)delta_ns, delta_pct);
}
static int join_thread(pthread_t thread, const struct timespec *deadline,
int *thread_err)
{
void *result;
int err;
err = pthread_timedjoin_np(thread, &result, deadline);
if (err)
return err;
*thread_err = (int)(uintptr_t)result;
return 0;
}
static void set_join_deadline(struct timespec *deadline)
{
clock_gettime(CLOCK_REALTIME, deadline);
deadline->tv_sec += JOIN_TIMEOUT_MS / 1000;
deadline->tv_nsec += (JOIN_TIMEOUT_MS % 1000) * 1000000;
if (deadline->tv_nsec >= 1000000000) {
deadline->tv_sec++;
deadline->tv_nsec -= 1000000000;
}
}
static enum scx_test_status setup(void **ctx)
{
struct enq_blocked *skel;
u64 flag;
skel = enq_blocked__open();
SCX_FAIL_IF(!skel, "Failed to open skel");
SCX_ENUM_INIT(skel);
flag = SCX_OPS_ENQ_BLOCKED;
if (!flag) {
enq_blocked__destroy(skel);
fprintf(stderr, "SKIP: SCX_OPS_ENQ_BLOCKED is unavailable\n");
return SCX_TEST_SKIP;
}
enq_blocked__destroy(skel);
*ctx = NULL;
return SCX_TEST_PASS;
}
static enum scx_test_status run_one(bool enq_blocked, bool cross_cpu,
struct run_result *result)
{
struct enq_blocked *skel;
struct thread_ctx thread_ctx = {};
struct contender_ctx *contender_ctxs = NULL;
struct bpf_link *link = NULL;
pthread_t owner, donor, *contenders = NULL;
struct timespec join_deadline;
cpu_set_t allowed_cpus;
bool module_loaded = false;
bool owner_started = false, donor_started = false;
bool join_timed_out = false;
bool proxy_enabled;
enum scx_test_status status = SCX_TEST_PASS;
int cpu, donor_pid, donor_state, err, thread_err;
size_t i, nr_contenders, nr_contenders_started = 0;
size_t nr_contenders_joined = 0;
u64 nr_blocked, nr_blocked_donor_cpu, nr_blocked_owner_cpu;
u64 nr_blocked_other_cpu, nr_blocked_wakeups;
struct enq_blocked_stats stats;
err = select_test_cpus(cross_cpu, &allowed_cpus, &thread_ctx.donor_cpu,
&thread_ctx.owner_cpu);
if (err == -EAGAIN) {
fprintf(stderr, "SKIP: cross-CPU case requires two allowed CPUs\n");
return SCX_TEST_SKIP;
}
if (err) {
SCX_ERR("Failed to select test CPUs (%d)", -err);
return SCX_TEST_FAIL;
}
nr_contenders = CPU_COUNT(&allowed_cpus);
contenders = calloc(nr_contenders, sizeof(*contenders));
contender_ctxs = calloc(nr_contenders, sizeof(*contender_ctxs));
if (!contenders || !contender_ctxs) {
SCX_ERR("Failed to allocate %zu contender threads", nr_contenders);
status = SCX_TEST_FAIL;
goto out_contenders;
}
skel = enq_blocked__open();
if (!skel) {
SCX_ERR("Failed to open skel");
status = SCX_TEST_FAIL;
goto out_contenders;
}
SCX_ENUM_INIT(skel);
skel->struct_ops.enq_blocked_ops->flags =
SCX_OPS_ENQ_LAST |
(enq_blocked ? SCX_OPS_ENQ_BLOCKED : 0);
if (enq_blocked__load(skel)) {
SCX_ERR("Failed to load skel");
status = SCX_TEST_FAIL;
goto out_skel;
}
err = load_test_module(&module_loaded);
if (err == -EPERM || err == -ENOENT) {
fprintf(stderr, "SKIP: cannot load mutex fixture (%d)\n", -err);
status = SCX_TEST_SKIP;
goto out_skel;
}
if (err) {
SCX_ERR("Failed to load mutex fixture (%d)", -err);
status = SCX_TEST_FAIL;
goto out_skel;
}
thread_ctx.fd = open(DEVICE_PATH, O_RDONLY | O_CLOEXEC);
if (thread_ctx.fd < 0) {
SCX_ERR("Failed to open %s (%d)", DEVICE_PATH, errno);
status = SCX_TEST_FAIL;
goto out_module;
}
err = ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED);
if (err < 0) {
SCX_ERR("Failed to query proxy-exec support (%d)", errno);
status = SCX_TEST_FAIL;
goto out_fd;
}
proxy_enabled = err && cmdline_bool("sched_proxy_exec", true);
if (!proxy_enabled) {
fprintf(stderr, "SKIP: proxy execution is not enabled\n");
status = SCX_TEST_SKIP;
goto out_fd;
}
if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) {
SCX_ERR("Failed to reset mutex statistics (%d)", errno);
status = SCX_TEST_FAIL;
goto out_fd;
}
if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PREP_ATTACH)) {
SCX_ERR("Failed to prepare scheduler attachment (%d)", errno);
status = SCX_TEST_FAIL;
goto out_fd;
}
err = pthread_create(&owner, NULL, owner_fn, &thread_ctx);
if (err) {
SCX_ERR("Failed to create owner thread (%d)", err);
status = SCX_TEST_FAIL;
goto out;
}
owner_started = true;
err = pthread_create(&donor, NULL, donor_fn, &thread_ctx);
if (err) {
SCX_ERR("Failed to create donor thread (%d)", err);
status = SCX_TEST_FAIL;
goto out;
}
donor_started = true;
if (!wait_for_pid(&thread_ctx.donor_pid)) {
SCX_ERR("Timed out waiting for donor thread");
status = SCX_TEST_FAIL;
goto out;
}
donor_pid = atomic_load_explicit(&thread_ctx.donor_pid,
memory_order_acquire);
skel->bss->donor_pid = donor_pid;
skel->data->donor_cpu = thread_ctx.donor_cpu;
skel->data->owner_cpu = thread_ctx.owner_cpu;
atomic_store_explicit(&thread_ctx.start_donor, true,
memory_order_release);
donor_state = ENQ_BLOCKED_DONOR_SLEEPING;
if (proxy_enabled)
donor_state |= ENQ_BLOCKED_DONOR_ON_RQ;
err = wait_for_donor_state(&thread_ctx, donor_state);
if (err < 0) {
SCX_ERR("Donor did not block before scheduler attachment (%d)", -err);
status = SCX_TEST_FAIL;
goto out;
}
link = bpf_map__attach_struct_ops(skel->maps.enq_blocked_ops);
if (!link) {
SCX_ERR("Failed to attach scheduler");
status = SCX_TEST_FAIL;
goto out;
}
/* Scheduler ownership changes start from a fully blocked donor. */
donor_state = ENQ_BLOCKED_DONOR_SLEEPING;
err = wait_for_donor_state(&thread_ctx, donor_state);
if (err < 0) {
SCX_ERR("Unexpected donor state after scheduler attachment (%d)",
-err);
status = SCX_TEST_FAIL;
goto out;
}
if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE)) {
SCX_ERR("Failed to complete scheduler attachment (%d)", errno);
status = SCX_TEST_FAIL;
goto out;
}
i = 0;
for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
if (!CPU_ISSET(cpu, &allowed_cpus))
continue;
contender_ctxs[i].thread_ctx = &thread_ctx;
contender_ctxs[i].cpu = cpu;
atomic_init(&contender_ctxs[i].status, 0);
err = pthread_create(&contenders[i], NULL, contender_fn,
&contender_ctxs[i]);
if (err) {
SCX_ERR("Failed to create contender for CPU %d (%d)",
cpu, err);
status = SCX_TEST_FAIL;
goto out;
}
nr_contenders_started++;
i++;
}
err = wait_for_contenders(contender_ctxs, nr_contenders);
if (err != 1) {
SCX_ERR("Contender threads failed (%d)", -err);
status = SCX_TEST_FAIL;
goto out;
}
/*
* The first trial spans scheduler attachment and validates the state
* transition, but including it would skew scheduling latency. Exclude it
* from both the mutex and BPF enqueue measurements.
*/
if (!wait_for_donor(&thread_ctx, NR_WARMUP_TRIALS)) {
SCX_ERR("Timed out waiting for warm-up trial");
status = SCX_TEST_FAIL;
goto out;
}
if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) {
SCX_ERR("Failed to reset mutex statistics after warm-up (%d)",
errno);
status = SCX_TEST_FAIL;
goto out;
}
skel->bss->nr_blocked_enqueues = 0;
skel->bss->nr_blocked_enqueues_donor_cpu = 0;
skel->bss->nr_blocked_enqueues_owner_cpu = 0;
skel->bss->nr_blocked_enqueues_other_cpu = 0;
skel->bss->nr_blocked_wakeups = 0;
atomic_store_explicit(&thread_ctx.measurement_ready, true,
memory_order_release);
out:
ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE);
if (status != SCX_TEST_PASS) {
atomic_store_explicit(&thread_ctx.abort, true, memory_order_release);
atomic_store_explicit(&thread_ctx.start_donor, true,
memory_order_release);
atomic_store_explicit(&thread_ctx.measurement_ready, true,
memory_order_release);
}
set_join_deadline(&join_deadline);
if (donor_started) {
err = join_thread(donor, &join_deadline, &thread_err);
if (err == ETIMEDOUT) {
SCX_ERR("Timed out waiting for donor thread");
join_timed_out = true;
status = SCX_TEST_FAIL;
} else if (err) {
SCX_ERR("Failed to join donor thread (%d)", err);
status = SCX_TEST_FAIL;
} else {
donor_started = false;
if (thread_err) {
SCX_ERR("Donor thread failed (%d)", thread_err);
status = SCX_TEST_FAIL;
}
}
}
if (!join_timed_out && owner_started) {
err = join_thread(owner, &join_deadline, &thread_err);
if (err == ETIMEDOUT) {
SCX_ERR("Timed out waiting for owner thread");
join_timed_out = true;
status = SCX_TEST_FAIL;
} else if (err) {
SCX_ERR("Failed to join owner thread (%d)", err);
status = SCX_TEST_FAIL;
} else {
owner_started = false;
if (thread_err) {
SCX_ERR("Owner thread failed (%d)", thread_err);
status = SCX_TEST_FAIL;
}
}
}
atomic_store_explicit(&thread_ctx.stop_contender, true,
memory_order_release);
for (i = 0; !join_timed_out && i < nr_contenders_started; i++) {
err = join_thread(contenders[i], &join_deadline, &thread_err);
if (err == ETIMEDOUT) {
SCX_ERR("Timed out waiting for contender on CPU %d",
contender_ctxs[i].cpu);
join_timed_out = true;
status = SCX_TEST_FAIL;
} else if (err) {
SCX_ERR("Failed to join contender on CPU %d (%d)",
contender_ctxs[i].cpu, err);
status = SCX_TEST_FAIL;
} else {
nr_contenders_joined++;
if (thread_err) {
SCX_ERR("Contender on CPU %d failed (%d)",
contender_ctxs[i].cpu, thread_err);
status = SCX_TEST_FAIL;
}
}
}
/* Restore the fair scheduler before waiting for any stranded thread. */
if (join_timed_out) {
atomic_store_explicit(&thread_ctx.abort, true,
memory_order_release);
if (link) {
bpf_link__destroy(link);
link = NULL;
}
if (donor_started)
pthread_join(donor, NULL);
if (owner_started)
pthread_join(owner, NULL);
for (i = nr_contenders_joined;
i < nr_contenders_started; i++)
pthread_join(contenders[i], NULL);
}
if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_GET_STATS, &stats)) {
SCX_ERR("Failed to read mutex statistics (%d)", errno);
status = SCX_TEST_FAIL;
} else {
result->stats = stats;
printf("\n[topology=%s SCX_OPS_ENQ_BLOCKED=%s]\n",
cross_cpu ? "cross-cpu" : "same-cpu",
enq_blocked ? "enabled" : "disabled");
printf(" proxy_exec=%s\n",
proxy_enabled ? "enabled" : "disabled");
printf(" donor_cpu=%d\n", thread_ctx.donor_cpu);
printf(" owner_cpu=%d\n", thread_ctx.owner_cpu);
printf(" nr_contenders=%zu\n", nr_contenders);
printf(" measured_trials=%d\n", NR_MEASURED_TRIALS);
printf(" owner_nice=%d\n", OWNER_NICE);
printf(" donor_nice=%d\n", DONOR_NICE);
printf(" contender_nice=%d\n", CONTENDER_NICE);
print_avg_time("mutex_hold", stats.hold_time_ns, stats.nr_holds);
print_avg_time("mutex_wait", stats.wait_time_ns, stats.nr_waits);
if (stats.nr_holds != NR_MEASURED_TRIALS ||
stats.nr_waits != NR_MEASURED_TRIALS) {
SCX_ERR("Expected %d measured trials, got %llu holds and %llu waits",
NR_MEASURED_TRIALS,
(unsigned long long)stats.nr_holds,
(unsigned long long)stats.nr_waits);
status = SCX_TEST_FAIL;
}
}
nr_blocked = skel->bss->nr_blocked_enqueues;
nr_blocked_donor_cpu = skel->bss->nr_blocked_enqueues_donor_cpu;
nr_blocked_owner_cpu = skel->bss->nr_blocked_enqueues_owner_cpu;
nr_blocked_other_cpu = skel->bss->nr_blocked_enqueues_other_cpu;
nr_blocked_wakeups = skel->bss->nr_blocked_wakeups;
result->nr_blocked_enqueues = nr_blocked;
result->nr_blocked_enqueues_donor_cpu = nr_blocked_donor_cpu;
result->nr_blocked_enqueues_owner_cpu = nr_blocked_owner_cpu;
result->nr_blocked_enqueues_other_cpu = nr_blocked_other_cpu;
result->nr_blocked_wakeups = nr_blocked_wakeups;
printf(" nr_blocked_enqueues=%llu\n",
(unsigned long long)nr_blocked);
printf(" nr_blocked_enqueues_donor_cpu=%llu\n",
(unsigned long long)nr_blocked_donor_cpu);
printf(" nr_blocked_enqueues_owner_cpu=%llu\n",
(unsigned long long)nr_blocked_owner_cpu);
printf(" nr_blocked_enqueues_other_cpu=%llu\n",
(unsigned long long)nr_blocked_other_cpu);
printf(" nr_blocked_wakeups=%llu\n",
(unsigned long long)nr_blocked_wakeups);
if (status == SCX_TEST_PASS) {
if (enq_blocked && proxy_enabled && !nr_blocked) {
SCX_ERR("ops.enqueue() did not receive the blocked donor");
status = SCX_TEST_FAIL;
} else if ((!enq_blocked || !proxy_enabled) && nr_blocked) {
SCX_ERR("ops.enqueue() unexpectedly received %llu blocked donors",
(unsigned long long)nr_blocked);
status = SCX_TEST_FAIL;
} else if (nr_blocked_wakeups) {
SCX_ERR("Ordinary wakeups received %llu blocked enqueue flags",
(unsigned long long)nr_blocked_wakeups);
status = SCX_TEST_FAIL;
} else if (nr_blocked_other_cpu) {
SCX_ERR("Blocked donor had %llu enqueues on unexpected CPUs",
(unsigned long long)nr_blocked_other_cpu);
status = SCX_TEST_FAIL;
}
}
if (UEI_EXITED(skel, uei)) {
SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
(unsigned long long)skel->data->uei.kind,
(long long)skel->data->uei.exit_code);
status = SCX_TEST_FAIL;
}
if (link)
bpf_link__destroy(link);
out_fd:
close(thread_ctx.fd);
out_module:
unload_test_module(module_loaded);
out_skel:
enq_blocked__destroy(skel);
out_contenders:
free(contender_ctxs);
free(contenders);
return status;
}
static enum scx_test_status run_topology(bool cross_cpu)
{
struct run_result disabled = {}, enabled = {};
enum scx_test_status status;
status = run_one(false, cross_cpu, &disabled);
if (status != SCX_TEST_PASS)
return status;
status = run_one(true, cross_cpu, &enabled);
if (status != SCX_TEST_PASS)
return status;
printf("\n[topology=%s delta: enabled - disabled]\n",
cross_cpu ? "cross-cpu" : "same-cpu");
print_avg_delta("mutex_hold", disabled.stats.hold_time_ns,
disabled.stats.nr_holds, enabled.stats.hold_time_ns,
enabled.stats.nr_holds);
print_avg_delta("mutex_wait", disabled.stats.wait_time_ns,
disabled.stats.nr_waits, enabled.stats.wait_time_ns,
enabled.stats.nr_waits);
return SCX_TEST_PASS;
}
static enum scx_test_status run(void *ctx)
{
enum scx_test_status status;
(void)ctx;
status = run_topology(false);
if (status != SCX_TEST_PASS)
return status;
status = run_topology(true);
if (status == SCX_TEST_SKIP)
return SCX_TEST_PASS;
return status;
}
struct scx_test enq_blocked = {
.name = "enq_blocked",
.description = "Verify proxy donor admission under CPU-wide contention",
.setup = setup,
.run = run,
};
REGISTER_SCX_TEST(&enq_blocked)
tools/testing/selftests/sched_ext/enq_blocked.h
+28
View File
@@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */
#ifndef __ENQ_BLOCKED_H
#define __ENQ_BLOCKED_H
#include <linux/ioctl.h>
#include <linux/types.h>
struct enq_blocked_stats {
__u64 hold_time_ns;
__u64 wait_time_ns;
__u64 nr_holds;
__u64 nr_waits;
};
#define ENQ_BLOCKED_IOCTL_OWNER _IO('s', 1)
#define ENQ_BLOCKED_IOCTL_DONOR _IO('s', 2)
#define ENQ_BLOCKED_IOCTL_RESET_STATS _IO('s', 3)
#define ENQ_BLOCKED_IOCTL_GET_STATS _IOR('s', 4, struct enq_blocked_stats)
#define ENQ_BLOCKED_IOCTL_PREP_ATTACH _IO('s', 5)
#define ENQ_BLOCKED_IOCTL_ATTACH_DONE _IO('s', 6)
#define ENQ_BLOCKED_IOCTL_DONOR_STATE _IO('s', 7)
#define ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED _IO('s', 8)
#define ENQ_BLOCKED_DONOR_SLEEPING (1U << 0)
#define ENQ_BLOCKED_DONOR_ON_RQ (1U << 1)
#endif /* __ENQ_BLOCKED_H */
tools/testing/selftests/sched_ext/exit.c
+6 -2
View File
@@ -43,8 +43,12 @@ static enum scx_test_status run(void *ctx)
}
/* Assumes uei.kind is written last */
while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
sched_yield();
if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) {
SCX_ERR("Timed out waiting for scheduler to exit");
bpf_link__destroy(link);
exit__destroy(skel);
return SCX_TEST_FAIL;
}
SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF));
SCX_EQ(skel->data->uei.exit_code, tc);
tools/testing/selftests/sched_ext/hotplug.c
+31 -15
View File
@@ -21,15 +21,17 @@ static bool is_cpu_online(void)
return file_read_long(online_path) > 0;
}
static void toggle_online_status(bool online)
static int toggle_online_status(bool online)
{
long val = online ? 1 : 0;
int ret;
ret = file_write_long(online_path, val);
if (ret != 0)
fprintf(stderr, "Failed to bring CPU %s (%s)",
fprintf(stderr, "Failed to bring CPU %s (%s)\n",
online ? "online" : "offline", strerror(errno));
return ret;
}
static enum scx_test_status setup(void **ctx)
@@ -44,6 +46,7 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined)
{
struct hotplug *skel;
struct bpf_link *link;
enum scx_test_status status = SCX_TEST_FAIL;
long kind, code;
SCX_ASSERT(is_cpu_online());
@@ -54,8 +57,8 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined)
SCX_FAIL_IF(hotplug__load(skel), "Failed to load skel");
/* Testing the offline -> online path, so go offline before starting */
if (onlining)
toggle_online_status(0);
if (onlining && toggle_online_status(0))
goto out_destroy_skel;
if (cbs_defined) {
kind = SCX_KIND_VAL(SCX_EXIT_UNREG_BPF);
@@ -79,28 +82,34 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined)
return SCX_TEST_FAIL;
}
toggle_online_status(onlining ? 1 : 0);
if (toggle_online_status(onlining ? 1 : 0))
goto out_destroy_link;
while (!UEI_EXITED(skel, uei))
sched_yield();
if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) {
SCX_ERR("Timed out waiting for scheduler to exit");
goto out_destroy_link;
}
SCX_EQ(skel->data->uei.kind, kind);
SCX_EQ(UEI_REPORT(skel, uei), code);
if (!onlining)
toggle_online_status(1);
if (!onlining && toggle_online_status(1))
goto out_destroy_link;
status = SCX_TEST_PASS;
out_destroy_link:
bpf_link__destroy(link);
out_destroy_skel:
hotplug__destroy(skel);
return SCX_TEST_PASS;
return status;
}
static enum scx_test_status test_hotplug_attach(void)
{
struct hotplug *skel;
struct bpf_link *link;
enum scx_test_status status = SCX_TEST_PASS;
enum scx_test_status status = SCX_TEST_FAIL;
long kind, code;
SCX_ASSERT(is_cpu_online());
@@ -115,14 +124,18 @@ static enum scx_test_status test_hotplug_attach(void)
* Take the CPU offline to increment the global hotplug seq, which
* should cause attach to fail due to us setting the hotplug seq above
*/
toggle_online_status(0);
if (toggle_online_status(0))
goto out_destroy_skel;
link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops);
toggle_online_status(1);
if (toggle_online_status(1))
goto out_destroy_link;
SCX_ASSERT(link);
while (!UEI_EXITED(skel, uei))
sched_yield();
if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) {
SCX_ERR("Timed out waiting for scheduler to exit");
goto out_destroy_link;
}
kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN);
code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) |
@@ -130,7 +143,10 @@ static enum scx_test_status test_hotplug_attach(void)
SCX_EQ(skel->data->uei.kind, kind);
SCX_EQ(UEI_REPORT(skel, uei), code);
status = SCX_TEST_PASS;
out_destroy_link:
bpf_link__destroy(link);
out_destroy_skel:
hotplug__destroy(skel);
return status;
tools/testing/selftests/sched_ext/init_enable_count.bpf.c
+8
View File
@@ -15,6 +15,8 @@ char _license[] SEC("license") = "GPL";
u64 init_task_cnt, exit_task_cnt, enable_cnt, disable_cnt;
u64 init_fork_cnt, init_transition_cnt;
UEI_DEFINE(uei);
s32 BPF_STRUCT_OPS_SLEEPABLE(cnt_init_task, struct task_struct *p,
struct scx_init_task_args *args)
{
@@ -43,11 +45,17 @@ void BPF_STRUCT_OPS(cnt_disable, struct task_struct *p)
__sync_fetch_and_add(&disable_cnt, 1);
}
void BPF_STRUCT_OPS(cnt_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops init_enable_count_ops = {
.init_task = (void *) cnt_init_task,
.exit_task = (void *) cnt_exit_task,
.enable = (void *) cnt_enable,
.disable = (void *) cnt_disable,
.exit = (void *) cnt_exit,
.name = "init_enable_count",
};
tools/testing/selftests/sched_ext/init_enable_count.c
+13
View File
@@ -5,6 +5,7 @@
* Copyright (c) 2023 Tejun Heo <tj@kernel.org>
*/
#include <signal.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <sched.h>
@@ -72,7 +73,17 @@ static enum scx_test_status run_test(bool global)
close(pipe_fds[1]);
signal(SIGCHLD, SIG_DFL);
SCX_ASSERT_ALIVE(skel, uei);
bpf_link__destroy(link);
/*
* The clean unregister above is recorded in uei and would
* otherwise trip the liveness check of the second cycle, which
* reuses this skeleton.
*/
memset(&skel->data->uei, 0, sizeof(skel->data->uei));
SCX_GE(skel->bss->init_task_cnt, num_pre_forks);
SCX_GE(skel->bss->exit_task_cnt, num_pre_forks);
@@ -124,6 +135,8 @@ static enum scx_test_status run_test(bool global)
status);
}
SCX_ASSERT_ALIVE(skel, uei);
bpf_link__destroy(link);
SCX_GE(skel->bss->init_task_cnt, 2 * num_children);
tools/testing/selftests/sched_ext/kick.bpf.c
+164
View File
@@ -0,0 +1,164 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*/
#include <scx/common.bpf.h>
#include "kick_test.h"
char _license[] SEC("license") = "GPL";
const volatile u32 scenario;
const volatile s32 victim_pid;
const volatile s32 challenger_pid;
const volatile s32 target_cpu;
const volatile s32 lazy_resched_override = -1;
u32 state;
u64 slice_before;
u64 slice_at_resched;
s32 resched_tif;
u64 nr_wait_callbacks;
bool enq_preempt_issued;
UEI_DEFINE(uei);
static bool is_trace_scenario(void)
{
return scenario <= TICK_EXPIRY;
}
void BPF_STRUCT_OPS(kick_enqueue, struct task_struct *p, u64 enq_flags)
{
switch (scenario) {
case ENQ_BOTH:
if (p->pid == victim_pid) {
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_INF, enq_flags);
} else if (p->pid == challenger_pid && state == KICK_STATE_QUEUED) {
enq_preempt_issued = true;
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL,
enq_flags | SCX_ENQ_PREEMPT |
SCX_ENQ_PREEMPT_LAZY);
} else {
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
}
goto kick_last;
case INVALID_KICK_IDLE:
scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_PREEMPT_LAZY | SCX_KICK_IDLE);
break;
case INVALID_KICK_UNKNOWN:
scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 1LLU << 63);
break;
}
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
kick_last:
if (enq_flags & SCX_ENQ_LAST)
scx_bpf_kick_cpu(target_cpu, SCX_KICK_IDLE);
}
static void set_lazy_resched_override(struct task_struct *p)
{
if (p->pid == victim_pid && lazy_resched_override >= 0)
scx_bpf_task_set_lazy_resched(p, lazy_resched_override);
}
void BPF_STRUCT_OPS(kick_running, struct task_struct *p)
{
if (!is_trace_scenario() || p->pid != victim_pid)
return;
set_lazy_resched_override(p);
if (__sync_val_compare_and_swap(&state, KICK_STATE_ARMED, KICK_STATE_QUEUED) !=
KICK_STATE_ARMED)
return;
slice_before = p->scx.slice;
switch (scenario) {
case KICK_IMMEDIATE:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT);
break;
case KICK_LAZY:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY);
break;
case KICK_LAZY_THEN_IMMEDIATE:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY);
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT);
break;
case KICK_IMMEDIATE_THEN_LAZY:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT);
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY);
break;
case KICK_PLAIN_THEN_LAZY:
scx_bpf_kick_cpu(target_cpu, 0);
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY);
break;
case KICK_LAZY_THEN_PLAIN:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY);
scx_bpf_kick_cpu(target_cpu, 0);
break;
case KICK_BOTH:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY);
break;
case KICK_LAZY_WAIT:
scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY | SCX_KICK_WAIT);
break;
case ENQ_BOTH:
/* The userspace controller wakes a competing task. */
break;
case TICK_EXPIRY:
/* no kick: the slice runs out at the tick */
break;
}
}
SEC("fexit/__resched_curr")
int BPF_PROG(kick_need_resched, struct rq *rq, int tif)
{
struct task_struct *task = BPF_CORE_READ(rq, curr);
u64 slice;
if (!task || BPF_CORE_READ(task, pid) != victim_pid ||
BPF_CORE_READ(rq, cpu) != target_cpu || state != KICK_STATE_QUEUED)
return 0;
if (scenario == ENQ_BOTH && !enq_preempt_issued)
return 0;
slice = BPF_CORE_READ(task, scx.slice);
if (slice)
return 0;
slice_at_resched = slice;
resched_tif = tif;
state = KICK_STATE_RESCHED;
return 0;
}
SEC("fentry/kick_sync_wait_bal_cb")
int BPF_PROG(kick_wait_callback, struct rq *rq)
{
if (scenario == KICK_LAZY_WAIT && BPF_CORE_READ(rq, cpu) == target_cpu)
__sync_fetch_and_add(&nr_wait_callbacks, 1);
return 0;
}
void BPF_STRUCT_OPS(kick_stopping, struct task_struct *p, bool runnable)
{
if (p->pid == victim_pid && state == KICK_STATE_RESCHED)
state = KICK_STATE_DONE;
}
void BPF_STRUCT_OPS(kick_exit, struct scx_exit_info *ei)
{
UEI_RECORD(uei, ei);
}
SEC(".struct_ops.link")
struct sched_ext_ops kick_ops = {
.enqueue = (void *)kick_enqueue,
.running = (void *)kick_running,
.stopping = (void *)kick_stopping,
.exit = (void *)kick_exit,
.flags = SCX_OPS_ENQ_LAST,
.name = "kick",
};
tools/testing/selftests/sched_ext/kick.c
+521
View File
@@ -0,0 +1,521 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*/
#define _GNU_SOURCE
#include <bpf/bpf.h>
#include <errno.h>
#include <linux/sched.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>
#include <scx/common.h>
#include "kick.bpf.skel.h"
#include "kick_test.h"
#include "scx_test.h"
#include "util.h"
#define WAIT_LOOPS 3000
struct observation {
u64 slice_before;
u64 slice_at_resched;
u64 nr_wait_callbacks;
s32 resched_tif;
};
struct kick_ctx {
cpu_set_t original_mask;
int target_cpu;
};
static bool enum_supported(const char *type, const char *name)
{
u64 value;
return __COMPAT_read_enum(type, name, &value);
}
static enum scx_test_status setup_controller(void **ctx_ptr)
{
struct kick_ctx *ctx;
cpu_set_t controller_mask;
int cpu, controller_cpu = -1;
ctx = calloc(1, sizeof(*ctx));
SCX_FAIL_IF(!ctx, "Failed to allocate context");
if (sched_getaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask)) {
free(ctx);
SCX_FAIL("Failed to get affinity (%d)", errno);
}
for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
if (!CPU_ISSET(cpu, &ctx->original_mask))
continue;
if (controller_cpu < 0) {
controller_cpu = cpu;
} else {
ctx->target_cpu = cpu;
break;
}
}
if (cpu == CPU_SETSIZE) {
printf("SKIP: two allowed CPUs are required\n");
free(ctx);
return SCX_TEST_SKIP;
}
CPU_ZERO(&controller_mask);
CPU_SET(controller_cpu, &controller_mask);
if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) {
free(ctx);
SCX_FAIL("Failed to pin controller to CPU %d (%d)", controller_cpu, errno);
}
*ctx_ptr = ctx;
return SCX_TEST_PASS;
}
static void cleanup_controller(void *ctx_ptr)
{
struct kick_ctx *ctx = ctx_ptr;
sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask);
free(ctx);
}
static bool wait_for_state(struct kick *skel, u32 wanted)
{
int i;
for (i = 0; i < WAIT_LOOPS; i++) {
if (__atomic_load_n(&skel->bss->state, __ATOMIC_ACQUIRE) == wanted)
return true;
if (UEI_EXITED(skel, uei))
return false;
usleep(1000);
}
return false;
}
static bool wait_for_counter(struct kick *skel, const u64 *counter, u64 wanted)
{
int i;
for (i = 0; i < WAIT_LOOPS; i++) {
if (__atomic_load_n(counter, __ATOMIC_ACQUIRE) >= wanted)
return true;
if (UEI_EXITED(skel, uei))
return false;
usleep(1000);
}
return false;
}
static enum scx_test_status trace_one(struct kick_ctx *ctx, u32 scenario, u64 ops_flags,
s32 lazy_resched_override, struct observation *obs)
{
struct bpf_link *ops_link = NULL;
struct kick *skel = NULL;
struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 };
struct scx_test_gated_worker victim;
enum scx_test_status ret = SCX_TEST_FAIL;
int cpu = ctx->target_cpu;
victim = scx_test_spawn_gated_worker(cpu, false);
if (victim.pid < 0) {
SCX_ERR("Failed to spawn victim");
return SCX_TEST_FAIL;
}
if (scenario == ENQ_BOTH) {
challenger = scx_test_spawn_gated_worker(cpu, false);
if (challenger.pid < 0) {
SCX_ERR("Failed to spawn enqueue challenger");
goto out;
}
}
skel = kick__open();
if (!skel) {
SCX_ERR("Failed to open scenario %u", scenario);
goto out;
}
SCX_ENUM_INIT(skel);
skel->rodata->scenario = scenario;
skel->rodata->victim_pid = victim.pid;
skel->rodata->challenger_pid = challenger.pid;
skel->rodata->target_cpu = cpu;
skel->rodata->lazy_resched_override = lazy_resched_override;
skel->struct_ops.kick_ops->flags |= ops_flags;
if (scenario != KICK_LAZY_WAIT)
bpf_program__set_autoload(skel->progs.kick_wait_callback, false);
if (kick__load(skel)) {
SCX_ERR("Failed to load scenario %u", scenario);
goto out;
}
bpf_map__set_autoattach(skel->maps.kick_ops, false);
if (kick__attach(skel)) {
SCX_ERR("Failed to attach __resched_curr tracer");
goto out;
}
skel->bss->state = KICK_STATE_ARMED;
ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops);
if (!ops_link) {
SCX_ERR("Failed to attach scenario %u", scenario);
goto out;
}
if (!scx_test_start_gated_worker(&victim)) {
SCX_ERR("Failed to start victim");
goto out;
}
if (scenario == ENQ_BOTH) {
if (!wait_for_state(skel, KICK_STATE_QUEUED)) {
SCX_ERR("Enqueue scenario did not arm");
goto out;
}
if (!scx_test_start_gated_worker(&challenger)) {
SCX_ERR("Failed to start enqueue challenger");
goto out;
}
}
if (!wait_for_state(skel, KICK_STATE_DONE)) {
SCX_ERR("Scenario %u stopped in state %u, exit kind %d", scenario, skel->bss->state,
skel->data->uei.kind);
goto out;
}
if (scenario == KICK_LAZY_WAIT &&
!wait_for_counter(skel, &skel->bss->nr_wait_callbacks, 1)) {
SCX_ERR("WAIT callback did not run");
goto out;
}
obs->slice_before = skel->bss->slice_before;
obs->slice_at_resched = skel->bss->slice_at_resched;
obs->nr_wait_callbacks = skel->bss->nr_wait_callbacks;
obs->resched_tif = skel->bss->resched_tif;
ret = SCX_TEST_PASS;
out:
scx_test_stop_gated_worker(&victim);
scx_test_stop_gated_worker(&challenger);
if (ops_link)
bpf_link__destroy(ops_link);
if (skel)
kick__destroy(skel);
return ret;
}
static bool observation_valid(const struct observation *obs)
{
return obs->slice_before > 0 && !obs->slice_at_resched;
}
static bool trace_target_supported(const char *name, enum bpf_attach_type attach_type)
{
return libbpf_find_vmlinux_btf_id(name, attach_type) > 0;
}
static enum scx_test_status setup_immediate(void **ctx)
{
if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT")) {
printf("SKIP: SCX_KICK_PREEMPT is not supported\n");
return SCX_TEST_SKIP;
}
if (!trace_target_supported("__resched_curr", BPF_TRACE_FEXIT)) {
printf("SKIP: __resched_curr is not available in BTF\n");
return SCX_TEST_SKIP;
}
return setup_controller(ctx);
}
static enum scx_test_status setup_lazy(void **ctx)
{
if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) {
printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n");
return SCX_TEST_SKIP;
}
return setup_immediate(ctx);
}
static enum scx_test_status setup_tick(void **ctx)
{
if (!enum_supported("scx_ops_flags", "SCX_OPS_LAZY_RESCHED")) {
printf("SKIP: SCX_OPS_LAZY_RESCHED is not supported\n");
return SCX_TEST_SKIP;
}
return setup_lazy(ctx);
}
static enum scx_test_status setup_coalesce(void **ctx)
{
if (!enum_supported("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY")) {
printf("SKIP: SCX_ENQ_PREEMPT_LAZY is not supported\n");
return SCX_TEST_SKIP;
}
if (!trace_target_supported("kick_sync_wait_bal_cb", BPF_TRACE_FENTRY)) {
printf("SKIP: kick_sync_wait_bal_cb is not available in BTF\n");
return SCX_TEST_SKIP;
}
return setup_lazy(ctx);
}
static enum scx_test_status setup_invalid(void **ctx)
{
if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) {
printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n");
return SCX_TEST_SKIP;
}
return setup_controller(ctx);
}
static enum scx_test_status run_immediate(void *ctx)
{
struct observation obs;
enum scx_test_status status;
status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &obs);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&obs));
return SCX_TEST_PASS;
}
static enum scx_test_status run_lazy(void *ctx)
{
struct observation immediate, lazy;
enum scx_test_status status;
int lazy_mode = scx_test_preempt_lazy_mode();
if (lazy_mode < 0) {
printf("SKIP: running kernel preemption mode is unavailable\n");
return SCX_TEST_SKIP;
}
status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&immediate));
status = trace_one(ctx, KICK_LAZY, 0, -1, &lazy);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&lazy));
if (lazy_mode > 0)
SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif,
"Lazy mode used immediate TIF %d", lazy.resched_tif);
else if (!lazy_mode)
SCX_EQ(lazy.resched_tif, immediate.resched_tif);
return SCX_TEST_PASS;
}
static enum scx_test_status run_coalesce(void *ctx)
{
struct observation immediate, lazy_first, immediate_first;
struct observation plain_first, lazy_then_plain;
struct observation both, lazy_wait, enq_both;
enum scx_test_status status;
if (scx_test_preempt_lazy_mode() != 1) {
printf("SKIP: kick coalescing requires active lazy preemption\n");
return SCX_TEST_SKIP;
}
status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&immediate));
status = trace_one(ctx, KICK_LAZY_THEN_IMMEDIATE, 0, -1, &lazy_first);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&lazy_first));
status = trace_one(ctx, KICK_IMMEDIATE_THEN_LAZY, 0, -1, &immediate_first);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&immediate_first));
SCX_EQ(lazy_first.resched_tif, immediate.resched_tif);
SCX_EQ(immediate_first.resched_tif, immediate.resched_tif);
/*
* A plain kick in the same batch doesn't clear the slice by itself. The
* lazy preemption must still expire it, and the plain kick must still
* reschedule immediately, whichever came first.
*/
status = trace_one(ctx, KICK_PLAIN_THEN_LAZY, 0, -1, &plain_first);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&plain_first));
status = trace_one(ctx, KICK_LAZY_THEN_PLAIN, 0, -1, &lazy_then_plain);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&lazy_then_plain));
SCX_EQ(plain_first.resched_tif, immediate.resched_tif);
SCX_EQ(lazy_then_plain.resched_tif, immediate.resched_tif);
/* Immediate kick and WAIT both take precedence over lazy preemption. */
status = trace_one(ctx, KICK_BOTH, 0, -1, &both);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&both));
status = trace_one(ctx, KICK_LAZY_WAIT, 0, -1, &lazy_wait);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&lazy_wait));
SCX_EQ(both.resched_tif, immediate.resched_tif);
SCX_EQ(lazy_wait.resched_tif, immediate.resched_tif);
SCX_GT(lazy_wait.nr_wait_callbacks, 0);
/* Immediate enqueue preemption likewise takes precedence over lazy. */
status = trace_one(ctx, ENQ_BOTH, 0, -1, &enq_both);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&enq_both));
SCX_EQ(enq_both.resched_tif, immediate.resched_tif);
return SCX_TEST_PASS;
}
/*
* A slice running out at the tick reschedules immediately by default and lazily
* with SCX_OPS_LAZY_RESCHED, the way fair.c expires a slice.
*/
static enum scx_test_status run_tick(void *ctx)
{
struct observation immediate, lazy, force_lazy, force_immediate;
enum scx_test_status status;
u64 lazy_flag;
int lazy_mode = scx_test_preempt_lazy_mode();
bool found;
if (lazy_mode < 0) {
printf("SKIP: running kernel preemption mode is unavailable\n");
return SCX_TEST_SKIP;
}
found = __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", &lazy_flag);
SCX_ASSERT(found);
status = trace_one(ctx, TICK_EXPIRY, 0, -1, &immediate);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&immediate));
status = trace_one(ctx, TICK_EXPIRY, lazy_flag, -1, &lazy);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&lazy));
if (lazy_mode > 0) {
SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif,
"Lazy slice expiry used immediate TIF %d", lazy.resched_tif);
status = trace_one(ctx, TICK_EXPIRY, 0, 1, &force_lazy);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&force_lazy));
status = trace_one(ctx, TICK_EXPIRY, lazy_flag, 0, &force_immediate);
SCX_EQ(status, SCX_TEST_PASS);
SCX_ASSERT(observation_valid(&force_immediate));
SCX_EQ(force_lazy.resched_tif, lazy.resched_tif);
SCX_EQ(force_immediate.resched_tif, immediate.resched_tif);
} else {
SCX_EQ(lazy.resched_tif, immediate.resched_tif);
}
return SCX_TEST_PASS;
}
static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario)
{
struct bpf_link *ops_link = NULL;
struct kick *skel = NULL;
struct scx_test_gated_worker victim;
enum scx_test_status ret = SCX_TEST_FAIL;
int cpu = ctx->target_cpu;
int i;
u64 error_kind;
error_kind = EXIT_KIND(SCX_EXIT_ERROR);
victim = scx_test_spawn_gated_worker(cpu, false);
if (victim.pid < 0)
return SCX_TEST_FAIL;
skel = kick__open();
if (!skel)
goto out;
SCX_ENUM_INIT(skel);
skel->rodata->scenario = scenario;
bpf_program__set_autoload(skel->progs.kick_need_resched, false);
bpf_program__set_autoload(skel->progs.kick_wait_callback, false);
if (kick__load(skel))
goto out;
ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops);
if (!ops_link || !scx_test_start_gated_worker(&victim))
goto out;
for (i = 0; i < WAIT_LOOPS; i++) {
if (UEI_EXITED(skel, uei) == error_kind) {
ret = SCX_TEST_PASS;
break;
}
usleep(1000);
}
out:
scx_test_stop_gated_worker(&victim);
if (ops_link)
bpf_link__destroy(ops_link);
if (skel)
kick__destroy(skel);
return ret;
}
static enum scx_test_status run_invalid(void *ctx)
{
enum scx_test_status status;
u32 scenario;
for (scenario = INVALID_KICK_IDLE;
scenario <= INVALID_KICK_UNKNOWN; scenario++) {
status = invalid_one(ctx, scenario);
SCX_EQ(status, SCX_TEST_PASS);
}
return SCX_TEST_PASS;
}
static struct scx_test kick_immediate = {
.name = "kick_immediate",
.description = "Trace immediate kick slice expiration and rescheduling",
.setup = setup_immediate,
.run = run_immediate,
.cleanup = cleanup_controller,
};
static struct scx_test kick_lazy = {
.name = "kick_lazy",
.description = "Trace lazy kick slice expiration and rescheduling",
.setup = setup_lazy,
.run = run_lazy,
.cleanup = cleanup_controller,
};
static struct scx_test kick_coalesce = {
.name = "kick_coalesce",
.description = "Verify lazy preemption coalesces with immediate requests",
.setup = setup_coalesce,
.run = run_coalesce,
.cleanup = cleanup_controller,
};
static struct scx_test kick_tick = {
.name = "kick_tick",
.description = "Trace slice expiry at the tick, immediate and lazy",
.setup = setup_tick,
.run = run_tick,
.cleanup = cleanup_controller,
};
static struct scx_test kick_invalid = {
.name = "kick_invalid",
.description = "Verify invalid kick flag combinations fail",
.setup = setup_invalid,
.run = run_invalid,
.cleanup = cleanup_controller,
};
__attribute__((constructor))
static void register_kick_tests(void)
{
scx_test_register(&kick_immediate);
scx_test_register(&kick_lazy);
scx_test_register(&kick_coalesce);
scx_test_register(&kick_tick);
scx_test_register(&kick_invalid);
}
tools/testing/selftests/sched_ext/kick_test.h
+29
View File
@@ -0,0 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */
#ifndef __KICK_TEST_H__
#define __KICK_TEST_H__
enum kick_scenario {
KICK_IMMEDIATE,
KICK_LAZY,
KICK_LAZY_THEN_IMMEDIATE,
KICK_IMMEDIATE_THEN_LAZY,
KICK_PLAIN_THEN_LAZY,
KICK_LAZY_THEN_PLAIN,
KICK_BOTH,
KICK_LAZY_WAIT,
ENQ_BOTH,
TICK_EXPIRY,
INVALID_KICK_IDLE,
INVALID_KICK_UNKNOWN,
};
enum kick_state {
KICK_STATE_IDLE,
KICK_STATE_ARMED,
KICK_STATE_QUEUED,
KICK_STATE_RESCHED,
KICK_STATE_DONE,
};
#endif /* __KICK_TEST_H__ */
tools/testing/selftests/sched_ext/maximal.bpf.c
+5 -1
View File
@@ -14,6 +14,8 @@ char _license[] SEC("license") = "GPL";
#define DSQ_ID 0
UEI_DEFINE(uei);
s32 BPF_STRUCT_OPS(maximal_select_cpu, struct task_struct *p, s32 prev_cpu,
u64 wake_flags)
{
@@ -132,7 +134,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(maximal_init)
}
void BPF_STRUCT_OPS(maximal_exit, struct scx_exit_info *info)
{}
{
UEI_RECORD(uei, info);
}
SEC(".struct_ops.link")
struct sched_ext_ops maximal_ops = {
tools/testing/selftests/sched_ext/maximal.c
+2
View File
@@ -35,6 +35,8 @@ static enum scx_test_status run(void *ctx)
link = bpf_map__attach_struct_ops(skel->maps.maximal_ops);
SCX_FAIL_IF(!link, "Failed to attach scheduler");
SCX_ASSERT_ALIVE(skel, uei);
bpf_link__destroy(link);
return SCX_TEST_PASS;
tools/testing/selftests/sched_ext/nohz_tick.bpf.c
+50 -7
View File
@@ -6,13 +6,22 @@
*/
#include <scx/common.bpf.h>
#include "nohz_tick_test.h"
char _license[] SEC("license") = "GPL";
const volatile s32 test_cpu;
bool finite_phase;
u32 phase;
s32 victim_pid;
s32 challenger_pid;
s32 trigger_pid;
u64 nr_inf_running;
u64 nr_finite_running;
u64 nr_finite_ticks;
u64 nr_lazy_victim_running;
u64 nr_lazy_enq_running;
u64 nr_lazy_kick_running;
u64 nr_lazy_ticks;
UEI_DEFINE(uei);
@@ -24,9 +33,31 @@ s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu,
void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags)
{
u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF;
u64 slice;
u64 dsq_id = SCX_DSQ_GLOBAL;
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags);
switch (phase) {
case NOHZ_PHASE_INF:
slice = SCX_SLICE_INF;
break;
case NOHZ_PHASE_FINITE:
slice = 1000000ULL;
break;
case NOHZ_PHASE_LAZY_ENQ:
case NOHZ_PHASE_LAZY_KICK:
dsq_id = SCX_DSQ_LOCAL;
slice = p->pid == victim_pid ? SCX_SLICE_INF : SCX_SLICE_DFL;
if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid)
enq_flags |= SCX_ENQ_PREEMPT_LAZY;
break;
default:
slice = SCX_SLICE_DFL;
break;
}
scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags);
if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == trigger_pid)
scx_bpf_kick_cpu(test_cpu, SCX_KICK_PREEMPT_LAZY);
if (enq_flags & SCX_ENQ_LAST)
scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE);
}
@@ -36,16 +67,28 @@ void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p)
if (bpf_get_smp_processor_id() != test_cpu)
return;
if (finite_phase)
if (phase == NOHZ_PHASE_FINITE)
__sync_fetch_and_add(&nr_finite_running, 1);
else
else if (phase == NOHZ_PHASE_INF)
__sync_fetch_and_add(&nr_inf_running, 1);
else if (p->pid == victim_pid)
__sync_fetch_and_add(&nr_lazy_victim_running, 1);
else if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid)
__sync_fetch_and_add(&nr_lazy_enq_running, 1);
else if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == challenger_pid)
__sync_fetch_and_add(&nr_lazy_kick_running, 1);
}
void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p)
{
if (bpf_get_smp_processor_id() == test_cpu && finite_phase)
if (bpf_get_smp_processor_id() != test_cpu)
return;
if (phase == NOHZ_PHASE_FINITE)
__sync_fetch_and_add(&nr_finite_ticks, 1);
else if ((phase == NOHZ_PHASE_LAZY_ENQ ||
phase == NOHZ_PHASE_LAZY_KICK) && p->pid == victim_pid)
__sync_fetch_and_add(&nr_lazy_ticks, 1);
}
void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei)
@@ -61,5 +104,5 @@ struct sched_ext_ops nohz_tick_ops = {
.tick = (void *)nohz_tick_tick,
.exit = (void *)nohz_tick_exit,
.name = "nohz_tick",
.timeout_ms = 1000U,
.timeout_ms = 5000U,
};
tools/testing/selftests/sched_ext/nohz_tick.c
+182 -27
View File
@@ -21,19 +21,24 @@
#include <scx/common.h>
#include "nohz_tick.bpf.skel.h"
#include "nohz_tick_test.h"
#include "scx_test.h"
#include "util.h"
#ifndef SCHED_EXT
#define SCHED_EXT 7
#endif
#define MIN_FINITE_TICKS 3
#define PHASE_TIMEOUT_MS 1000
#define PHASE_TIMEOUT_MS 5000
#define TICK_STOP_STABLE_MS 100
struct nohz_tick_ctx {
struct nohz_tick *skel;
cpu_set_t original_mask;
int test_cpu;
int housekeeping_cpu;
bool test_lazy;
};
static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
@@ -47,20 +52,21 @@ static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
return -1;
}
static int find_nohz_full_cpu(const cpu_set_t *allowed)
static int read_nohz_full_mask(cpu_set_t *mask)
{
char buf[4096], *cur, *end;
FILE *file;
int ret = 0;
file = fopen("/sys/devices/system/cpu/nohz_full", "r");
if (!file)
return -1;
return -errno;
if (!fgets(buf, sizeof(buf), file)) {
fclose(file);
return -1;
ret = ferror(file) ? -errno : -EINVAL;
goto out;
}
fclose(file);
CPU_ZERO(mask);
cur = buf;
while (*cur) {
long first, last;
@@ -73,25 +79,30 @@ static int find_nohz_full_cpu(const cpu_set_t *allowed)
errno = 0;
first = strtol(cur, &end, 10);
if (errno || end == cur || first < 0 || first >= CPU_SETSIZE)
return -1;
if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) {
ret = -EINVAL;
goto out;
}
cur = end;
last = first;
if (*cur == '-') {
cur++;
errno = 0;
last = strtol(cur, &end, 10);
if (errno || end == cur || last < first)
return -1;
if (errno || end == cur || last < first) {
ret = -EINVAL;
goto out;
}
cur = end;
}
cpu = first_allowed_cpu(allowed, first, last);
if (cpu >= 0)
return cpu;
for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++)
CPU_SET(cpu, mask);
}
return -1;
out:
fclose(file);
return ret;
}
static pid_t start_worker(int cpu)
@@ -146,24 +157,53 @@ static int pause_worker(pid_t pid)
return 0;
}
static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms)
static bool wait_for_counter(struct nohz_tick *skel, const u64 *counter, u64 value,
int timeout_ms)
{
int elapsed;
for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value)
return true;
if (UEI_EXITED(skel, uei))
return false;
usleep(1000);
}
return false;
}
static bool wait_for_tick_stop(struct nohz_tick *skel, const u64 *counter, int timeout_ms)
{
u64 prev = __atomic_load_n(counter, __ATOMIC_RELAXED);
int elapsed, stable = 0;
for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
u64 curr;
usleep(1000);
if (UEI_EXITED(skel, uei))
return false;
curr = __atomic_load_n(counter, __ATOMIC_RELAXED);
if (curr == prev) {
if (++stable >= TICK_STOP_STABLE_MS)
return true;
} else {
prev = curr;
stable = 0;
}
}
return false;
}
static enum scx_test_status setup(void **ctx_ptr)
{
struct nohz_tick_ctx *ctx;
cpu_set_t controller_mask;
int cpu;
cpu_set_t controller_mask, nohz_full_mask;
bool lazy_supported;
u64 enum_value, lazy_ops_flag;
int cpu, i, lazy_mode, ret;
ctx = calloc(1, sizeof(*ctx));
SCX_FAIL_IF(!ctx, "Failed to allocate context");
@@ -173,15 +213,26 @@ static enum scx_test_status setup(void **ctx_ptr)
SCX_FAIL("Failed to get affinity (%d)", errno);
}
cpu = find_nohz_full_cpu(&ctx->original_mask);
if (cpu < 0) {
ret = read_nohz_full_mask(&nohz_full_mask);
if (ret) {
fprintf(stderr, "SKIP: failed to read NOHZ_FULL mask (%d)\n", ret);
free(ctx);
return SCX_TEST_SKIP;
}
for (cpu = 0; cpu < CPU_SETSIZE; cpu++)
if (CPU_ISSET(cpu, &ctx->original_mask) && CPU_ISSET(cpu, &nohz_full_mask))
break;
if (cpu == CPU_SETSIZE) {
fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n");
free(ctx);
return SCX_TEST_SKIP;
}
controller_mask = ctx->original_mask;
CPU_CLR(cpu, &controller_mask);
for (i = 0; i < CPU_SETSIZE; i++)
if (CPU_ISSET(i, &nohz_full_mask))
CPU_CLR(i, &controller_mask);
if (CPU_COUNT(&controller_mask) == 0) {
fprintf(stderr, "SKIP: no housekeeping CPU available\n");
free(ctx);
@@ -189,6 +240,21 @@ static enum scx_test_status setup(void **ctx_ptr)
}
ctx->test_cpu = cpu;
ctx->housekeeping_cpu = first_allowed_cpu(&controller_mask, 0, CPU_SETSIZE - 1);
lazy_supported = __COMPAT_read_enum("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY",
&enum_value) &&
__COMPAT_read_enum("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY",
&enum_value) &&
__COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED",
&lazy_ops_flag);
lazy_mode = scx_test_preempt_lazy_mode();
ctx->test_lazy = lazy_supported && lazy_mode > 0;
if (lazy_supported && lazy_mode < 0)
fprintf(stderr,
"SKIP: kernel preemption mode unavailable; skipping lazy NOHZ_FULL phases\n");
else if (lazy_supported && !lazy_mode)
fprintf(stderr,
"SKIP: lazy preemption inactive; skipping lazy NOHZ_FULL phases\n");
ctx->skel = nohz_tick__open();
if (!ctx->skel) {
free(ctx);
@@ -199,6 +265,8 @@ static enum scx_test_status setup(void **ctx_ptr)
ctx->skel->rodata->test_cpu = cpu;
ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL |
SCX_OPS_ENQ_LAST;
if (lazy_supported)
ctx->skel->struct_ops.nohz_tick_ops->flags |= lazy_ops_flag;
if (nohz_tick__load(ctx->skel)) {
nohz_tick__destroy(ctx->skel);
free(ctx);
@@ -220,11 +288,15 @@ static enum scx_test_status run(void *ctx_ptr)
struct nohz_tick_ctx *ctx = ctx_ptr;
struct nohz_tick *skel = ctx->skel;
struct bpf_link *link = NULL;
struct scx_test_gated_worker victim = { .pid = -1, .start_fd = -1 };
struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 };
struct scx_test_gated_worker trigger = { .pid = -1, .start_fd = -1 };
enum scx_test_status status = SCX_TEST_FAIL;
pid_t finite_worker = -1;
pid_t inf_worker = -1;
u64 finite_running;
u64 finite_ticks;
u64 victim_running;
int ret;
link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops);
@@ -241,7 +313,7 @@ static enum scx_test_status run(void *ctx_ptr)
SCX_ERR("Failed to start infinite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_inf_running, 1,
if (!wait_for_counter(skel, &skel->bss->nr_inf_running, 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Infinite-slice worker was not scheduled");
goto out;
@@ -260,18 +332,18 @@ static enum scx_test_status run(void *ctx_ptr)
/*
* The next EXT task receives a finite slice and must restart the tick.
*/
__atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE);
__atomic_store_n(&skel->bss->phase, NOHZ_PHASE_FINITE, __ATOMIC_RELEASE);
finite_worker = start_worker(ctx->test_cpu);
if (finite_worker < 0) {
SCX_ERR("Failed to start finite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_running, 1,
if (!wait_for_counter(skel, &skel->bss->nr_finite_running, 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Finite-slice worker was not scheduled");
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Finite-slice worker received only %llu scheduler ticks",
(unsigned long long)skel->bss->nr_finite_ticks);
@@ -295,12 +367,12 @@ static enum scx_test_status run(void *ctx_ptr)
SCX_ERR("Failed to start second finite-slice worker (%d)", errno);
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_running,
if (!wait_for_counter(skel, &skel->bss->nr_finite_running,
finite_running + 1, PHASE_TIMEOUT_MS)) {
SCX_ERR("Second finite-slice worker was not scheduled");
goto out;
}
if (!wait_for_counter(&skel->bss->nr_finite_ticks,
if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks,
finite_ticks + MIN_FINITE_TICKS,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Second finite-slice worker received only %llu scheduler ticks",
@@ -308,8 +380,88 @@ static enum scx_test_status run(void *ctx_ptr)
finite_ticks));
goto out;
}
stop_worker(finite_worker);
finite_worker = -1;
stop_worker(inf_worker);
inf_worker = -1;
if (!ctx->test_lazy)
goto check_exit;
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
/*
* A lazy local enqueue must restart the tick after clearing the slice
* of an infinite-slice task on a full-dynticks CPU.
*/
__atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_ENQ, __ATOMIC_RELEASE);
victim = scx_test_spawn_gated_worker(ctx->test_cpu, true);
challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true);
if (victim.pid < 0 || challenger.pid < 0) {
SCX_ERR("Failed to spawn lazy-enqueue workers");
goto out;
}
skel->bss->victim_pid = victim.pid;
skel->bss->challenger_pid = challenger.pid;
if (!scx_test_start_gated_worker(&victim) ||
!wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Lazy-enqueue victim was not scheduled");
goto out;
}
if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) {
SCX_ERR("Tick did not stop before lazy enqueue");
goto out;
}
if (!scx_test_start_gated_worker(&challenger) ||
!wait_for_counter(skel, &skel->bss->nr_lazy_enq_running, 1, PHASE_TIMEOUT_MS)) {
SCX_ERR("Lazy enqueue made no progress on CPU %d", ctx->test_cpu);
goto out;
}
scx_test_stop_gated_worker(&victim);
scx_test_stop_gated_worker(&challenger);
/* Repeat with a lazy kick delivered from a housekeeping CPU. */
__atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_KICK, __ATOMIC_RELEASE);
victim = scx_test_spawn_gated_worker(ctx->test_cpu, true);
challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true);
trigger = scx_test_spawn_gated_worker(ctx->housekeeping_cpu, true);
if (victim.pid < 0 || challenger.pid < 0 || trigger.pid < 0) {
SCX_ERR("Failed to spawn lazy-kick workers");
goto out;
}
skel->bss->victim_pid = victim.pid;
skel->bss->challenger_pid = challenger.pid;
skel->bss->trigger_pid = trigger.pid;
victim_running = __atomic_load_n(&skel->bss->nr_lazy_victim_running,
__ATOMIC_RELAXED);
if (!scx_test_start_gated_worker(&victim) ||
!wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, victim_running + 1,
PHASE_TIMEOUT_MS)) {
SCX_ERR("Lazy-kick victim was not scheduled");
goto out;
}
if (!scx_test_start_gated_worker(&challenger)) {
SCX_ERR("Failed to start lazy-kick challenger");
goto out;
}
if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) {
SCX_ERR("Tick did not stop before lazy kick");
goto out;
}
if (__atomic_load_n(&skel->bss->nr_lazy_kick_running, __ATOMIC_RELAXED)) {
SCX_ERR("Lazy-kick challenger ran before the kick");
goto out;
}
if (!scx_test_start_gated_worker(&trigger) ||
!wait_for_counter(skel, &skel->bss->nr_lazy_kick_running, 1, PHASE_TIMEOUT_MS)) {
SCX_ERR("Lazy kick made no progress on CPU %d", ctx->test_cpu);
goto out;
}
scx_test_stop_gated_worker(&trigger);
scx_test_stop_gated_worker(&victim);
scx_test_stop_gated_worker(&challenger);
check_exit:
if (UEI_EXITED(skel, uei)) {
SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
(unsigned long long)skel->data->uei.kind,
(long long)skel->data->uei.exit_code);
@@ -321,6 +473,9 @@ static enum scx_test_status run(void *ctx_ptr)
(unsigned long long)skel->bss->nr_finite_ticks);
status = SCX_TEST_PASS;
out:
scx_test_stop_gated_worker(&trigger);
scx_test_stop_gated_worker(&victim);
scx_test_stop_gated_worker(&challenger);
stop_worker(finite_worker);
stop_worker(inf_worker);
if (link)
tools/testing/selftests/sched_ext/nohz_tick_test.h
+13
View File
@@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */
#ifndef __NOHZ_TICK_TEST_H__
#define __NOHZ_TICK_TEST_H__
enum nohz_phase {
NOHZ_PHASE_INF,
NOHZ_PHASE_FINITE,
NOHZ_PHASE_LAZY_ENQ,
NOHZ_PHASE_LAZY_KICK,
};
#endif /* __NOHZ_TICK_TEST_H__ */
tools/testing/selftests/sched_ext/prog_run.c
+5 -2
View File
@@ -55,8 +55,11 @@ static enum scx_test_status run(void *ctx)
}
/* Assumes uei.kind is written last */
while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
sched_yield();
if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) {
SCX_ERR("Timed out waiting for scheduler to exit");
status = SCX_TEST_FAIL;
goto out;
}
if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) {
SCX_ERR("Unexpected exit kind: %llu",
tools/testing/selftests/sched_ext/reload_loop.c
+41 -8
View File
@@ -4,6 +4,7 @@
* Copyright (c) 2024 David Vernet <dvernet@meta.com>
*/
#include <bpf/bpf.h>
#include <errno.h>
#include <pthread.h>
#include <scx/common.h>
#include <sys/wait.h>
@@ -11,8 +12,14 @@
#include "maximal.bpf.skel.h"
#include "scx_test.h"
enum {
RELOAD_ITERS = 1024,
RELOAD_THREADS = 2,
};
static struct maximal *skel;
static pthread_t threads[2];
static pthread_t threads[RELOAD_THREADS];
static int unexpected_errno;
bool force_exit = false;
@@ -31,32 +38,58 @@ static enum scx_test_status setup(void **ctx)
static void *do_reload_loop(void *arg)
{
u32 i;
u32 i, failures = 0;
for (i = 0; i < 1024 && !force_exit; i++) {
for (i = 0; i < RELOAD_ITERS && !force_exit; i++) {
struct bpf_link *link;
errno = 0;
link = bpf_map__attach_struct_ops(skel->maps.maximal_ops);
if (link)
bpf_link__destroy(link);
if (!link) {
int e = errno;
if (e != EBUSY)
__atomic_store_n(&unexpected_errno, e,
__ATOMIC_RELAXED);
failures++;
continue;
}
bpf_link__destroy(link);
}
return NULL;
return (void *)(unsigned long)failures;
}
static enum scx_test_status run(void *ctx)
{
int err;
unsigned long failures[RELOAD_THREADS] = {};
int err, i;
void *ret;
err = pthread_create(&threads[0], NULL, do_reload_loop, NULL);
SCX_FAIL_IF(err, "Failed to create thread 0");
err = pthread_create(&threads[1], NULL, do_reload_loop, NULL);
SCX_FAIL_IF(err, "Failed to create thread 1");
if (err) {
force_exit = true;
pthread_join(threads[0], NULL);
SCX_FAIL("Failed to create thread 1");
}
SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed");
failures[0] = (unsigned long)ret;
SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed");
failures[1] = (unsigned long)ret;
SCX_FAIL_IF(unexpected_errno, "unexpected attach errno %d",
unexpected_errno);
for (i = 0; i < RELOAD_THREADS; i++)
SCX_FAIL_IF(failures[i] == RELOAD_ITERS,
"thread %d: every attach failed (%lu EBUSY)",
i, failures[i]);
return SCX_TEST_PASS;
}
tools/testing/selftests/sched_ext/rt_stall.c
+42 -38
View File
@@ -17,7 +17,6 @@
#include <scx/common.h>
#include "rt_stall.bpf.skel.h"
#include "scx_test.h"
#include "../kselftest.h"
#define CORE_ID 0 /* CPU to pin tasks to */
#define RUN_TIME 5 /* How long to run the test in seconds */
@@ -35,15 +34,17 @@ static void signal_ready(int fd)
}
/* Wait for a child to signal readiness via a pipe */
static void wait_ready(int fd)
static bool wait_ready(int fd)
{
bool ready;
char c;
if (read(fd, &c, 1) != 1) {
ready = read(fd, &c, 1) == 1;
if (!ready)
perror("read from ready pipe");
exit(EXIT_FAILURE);
}
close(fd);
return ready;
}
/* Simple busy-wait function for test tasks */
@@ -151,13 +152,11 @@ static bool sched_stress_test(bool is_ext)
float ext_runtime, rt_runtime, actual_ratio;
int ext_pid, rt_pid;
int ext_ready[2], rt_ready[2];
ksft_print_header();
ksft_set_plan(1);
bool ret = false;
if (pipe(ext_ready) || pipe(rt_ready)) {
perror("pipe");
ksft_exit_fail();
return false;
}
/* Create and set up a EXT task */
@@ -172,7 +171,7 @@ static bool sched_stress_test(bool is_ext)
exit(0);
} else if (ext_pid < 0) {
perror("fork task");
ksft_exit_fail();
return false;
}
/* Create an RT task */
@@ -188,7 +187,7 @@ static bool sched_stress_test(bool is_ext)
exit(0);
} else if (rt_pid < 0) {
perror("fork for RT task");
ksft_exit_fail();
goto out_kill_ext;
}
/*
@@ -199,45 +198,47 @@ static bool sched_stress_test(bool is_ext)
*/
close(ext_ready[1]);
close(rt_ready[1]);
wait_ready(ext_ready[0]);
wait_ready(rt_ready[0]);
if (!wait_ready(ext_ready[0]) || !wait_ready(rt_ready[0]))
goto out_kill;
/* Let the processes run for the specified time */
sleep(RUN_TIME);
/* Get runtime for the EXT task */
ext_runtime = get_process_runtime(ext_pid);
if (ext_runtime == -1)
ksft_exit_fail_msg("Error getting runtime for %s task (PID %d)\n",
class_str, ext_pid);
ksft_print_msg("Runtime of %s task (PID %d) is %f seconds\n",
class_str, ext_pid, ext_runtime);
if (ext_runtime == -1) {
fprintf(stderr, "Failed to read %s task runtime\n", class_str);
goto out_kill;
}
/* Get runtime for the RT task */
rt_runtime = get_process_runtime(rt_pid);
if (rt_runtime == -1)
ksft_exit_fail_msg("Error getting runtime for RT task (PID %d)\n", rt_pid);
ksft_print_msg("Runtime of RT task (PID %d) is %f seconds\n", rt_pid, rt_runtime);
/* Kill the processes */
kill(ext_pid, SIGKILL);
kill(rt_pid, SIGKILL);
waitpid(ext_pid, NULL, 0);
waitpid(rt_pid, NULL, 0);
if (rt_runtime == -1) {
fprintf(stderr, "Failed to read RT task runtime\n");
goto out_kill;
}
/* Verify that the scx task got enough runtime */
actual_ratio = ext_runtime / (ext_runtime + rt_runtime);
ksft_print_msg("%s task got %.2f%% of total runtime\n",
class_str, actual_ratio * 100);
fprintf(stderr, "%s task ran %.3fs, RT task ran %.3fs (%.2f%% of runtime)\n",
class_str, ext_runtime, rt_runtime, actual_ratio * 100);
if (actual_ratio >= expected_min_ratio) {
ksft_test_result_pass("PASS: %s task got more than %.2f%% of runtime\n",
class_str, expected_min_ratio * 100);
return true;
if (actual_ratio < expected_min_ratio) {
fprintf(stderr, "%s task got less than %.2f%% of runtime\n",
class_str, expected_min_ratio * 100);
goto out_kill;
}
ksft_test_result_fail("FAIL: %s task got less than %.2f%% of runtime\n",
class_str, expected_min_ratio * 100);
return false;
ret = true;
out_kill:
kill(rt_pid, SIGKILL);
waitpid(rt_pid, NULL, 0);
out_kill_ext:
kill(ext_pid, SIGKILL);
waitpid(ext_pid, NULL, 0);
return ret;
}
static enum scx_test_status run(void *ctx)
@@ -263,14 +264,17 @@ static enum scx_test_status run(void *ctx)
link = bpf_map__attach_struct_ops(skel->maps.rt_stall_ops);
SCX_FAIL_IF(!link, "Failed to attach scheduler");
}
res = sched_stress_test(is_ext);
if (is_ext) {
SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE));
int exit_kind = skel->data->uei.kind;
bpf_link__destroy(link);
SCX_EQ(exit_kind, EXIT_KIND(SCX_EXIT_NONE));
}
if (!res)
ksft_exit_fail();
return SCX_TEST_FAIL;
}
return SCX_TEST_PASS;
tools/testing/selftests/sched_ext/runner.c
+1 -1
View File
@@ -243,7 +243,7 @@ int main(int argc, char **argv)
printf(" - %s\n", failed_tests[i]);
}
return failed > 0 ? 1 : 0;
return failed > 0 || exit_req ? 1 : 0;
}
void scx_test_register(struct scx_test *test)
tools/testing/selftests/sched_ext/scx_test.h
+51 -6
View File
@@ -18,8 +18,6 @@ enum scx_test_status {
SCX_TEST_FAIL,
};
#define EXIT_KIND(__ent) __COMPAT_ENUM_OR_ZERO("scx_exit_kind", #__ent)
struct scx_test {
/**
* name - The name of the testcase.
@@ -110,6 +108,41 @@ void scx_test_register(struct scx_test *test);
#define SCX_ASSERT(_x) SCX_FAIL_IF(!(_x), "Expected %s to be true (%lu)", \
#_x, (u64)(_x))
/*
* SCX_ASSERT_ALIVE - Fail the test if the scheduler under test exited
* on its own between attach and this check, e.g. from a runtime error.
* A scheduler that dies mid-test freezes whatever state the test is
* about to assert on, which can make the assertions pass vacuously.
* uei.msg carries the actual error text, so print it alongside the
* recorded kind and exit code.
*/
#define SCX_ASSERT_ALIVE(__skel, __uei) \
do { \
if (UEI_EXITED(__skel, __uei)) \
SCX_FAIL("Scheduler exited unexpectedly" \
" (kind=%llu code=%lld msg='%s')", \
(unsigned long long)(__skel)->data->__uei.kind, \
(long long)(__skel)->data->__uei.exit_code, \
(__skel)->data->__uei.msg); \
} while (0)
/*
* SCX_TIMED_WAIT_UEI - Wait for the scheduler under test to exit, i.e.
* its uei to be filled in, for up to @__ms iterations at ~1ms each.
* Returns true if the scheduler exited and false on timeout, so a
* scheduler that never exits fails the test instead of hanging it.
* The bound is iteration-based rather than wall-clock and stretches
* on a loaded system instead of failing early.
*/
#define SCX_TIMED_WAIT_UEI(__skel, __uei, __ms) \
({ \
s64 __left = (__ms); \
\
while (__left-- > 0 && !UEI_EXITED(__skel, __uei)) \
usleep(1000); \
UEI_EXITED(__skel, __uei) != 0; \
})
#define SCX_ECODE_VAL(__ecode) ({ \
u64 __val = 0; \
bool __found = false; \
@@ -119,13 +152,25 @@ void scx_test_register(struct scx_test *test);
(s64)__val; \
})
/*
* SCX_KIND_VAL - Resolve an scx_exit_kind enumerator from kernel BTF.
*
* Unlike __COMPAT_ENUM_OR_ZERO(), a missing enumerator fails the test
* loudly instead of silently resolving to 0 (== SCX_EXIT_NONE), which
* would make the comparison vacuous.
*/
#define SCX_KIND_VAL(__kind) ({ \
u64 __val = 0; \
bool __found = false; \
u64 __val = 0; \
\
__found = __COMPAT_read_enum("scx_exit_kind", #__kind, &__val); \
SCX_ASSERT(__found); \
if (!__COMPAT_read_enum("scx_exit_kind", #__kind, &__val)) \
SCX_FAIL("scx_exit_kind.%s not found in kernel BTF", #__kind); \
__val; \
})
/*
* EXIT_KIND - Alias of SCX_KIND_VAL() for resolving scx_exit_kind
* enumerator values, kept for readability at the EXIT_KIND() call sites.
*/
#define EXIT_KIND(__ent) SCX_KIND_VAL(__ent)
#endif // # __SCX_TEST_H__
tools/testing/selftests/sched_ext/test_modules/Makefile
+13
View File
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
KDIR ?= $(if $(O),$(O),$(realpath ../../../../..))
obj-m += scx_enq_blocked_test.o
all:
+$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) modules
clean:
+$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) clean
tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c
+195
View File
@@ -0,0 +1,195 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
*
* Kernel mutex fixture for the sched_ext SCX_OPS_ENQ_BLOCKED selftest.
*/
#include <linux/atomic.h>
#include <linux/fs.h>
#include <linux/jiffies.h>
#include <linux/ktime.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/sched.h>
#include <linux/uaccess.h>
#include "../enq_blocked.h"
#define DONOR_WAIT_TIMEOUT msecs_to_jiffies(2000)
#define ATTACH_WAIT_TIMEOUT msecs_to_jiffies(10000)
#define MUTEX_HOLD_TIME msecs_to_jiffies(200)
static DEFINE_MUTEX(test_mutex);
static DEFINE_SPINLOCK(donor_lock);
static struct task_struct *donor_task;
static atomic_t owner_ready = ATOMIC_INIT(0);
static atomic_t donor_started = ATOMIC_INIT(0);
static atomic_t attach_pending = ATOMIC_INIT(0);
static atomic_t attach_done = ATOMIC_INIT(0);
static atomic64_t hold_time_ns = ATOMIC64_INIT(0);
static atomic64_t wait_time_ns = ATOMIC64_INIT(0);
static atomic64_t nr_holds = ATOMIC64_INIT(0);
static atomic64_t nr_waits = ATOMIC64_INIT(0);
static long run_owner(void)
{
unsigned long timeout;
u64 start_ns;
long ret = 0;
atomic_set(&donor_started, 0);
mutex_lock(&test_mutex);
start_ns = ktime_get_ns();
atomic_set(&owner_ready, 1);
timeout = jiffies + DONOR_WAIT_TIMEOUT;
while (!atomic_read(&donor_started)) {
if (time_after(jiffies, timeout)) {
ret = -ETIMEDOUT;
goto out;
}
cond_resched();
}
if (atomic_xchg(&attach_pending, 0)) {
timeout = jiffies + ATTACH_WAIT_TIMEOUT;
while (!atomic_read(&attach_done)) {
if (time_after(jiffies, timeout)) {
ret = -ETIMEDOUT;
goto out;
}
cond_resched();
}
}
/* Keep yielding while the donor blocks on test_mutex. */
timeout = jiffies + MUTEX_HOLD_TIME;
while (time_before(jiffies, timeout))
cond_resched();
out:
atomic_set(&owner_ready, 0);
atomic64_add(ktime_get_ns() - start_ns, &hold_time_ns);
atomic64_inc(&nr_holds);
mutex_unlock(&test_mutex);
return ret;
}
static long run_donor(void)
{
unsigned long flags;
u64 start_ns;
if (!atomic_read(&owner_ready))
return -EAGAIN;
get_task_struct(current);
spin_lock_irqsave(&donor_lock, flags);
WARN_ON_ONCE(donor_task);
donor_task = current;
spin_unlock_irqrestore(&donor_lock, flags);
atomic_set(&donor_started, 1);
start_ns = ktime_get_ns();
mutex_lock(&test_mutex);
spin_lock_irqsave(&donor_lock, flags);
donor_task = NULL;
spin_unlock_irqrestore(&donor_lock, flags);
put_task_struct(current);
atomic64_add(ktime_get_ns() - start_ns, &wait_time_ns);
atomic64_inc(&nr_waits);
mutex_unlock(&test_mutex);
return 0;
}
static long get_donor_state(void)
{
struct task_struct *task;
unsigned long flags;
long state = 0;
spin_lock_irqsave(&donor_lock, flags);
task = donor_task;
if (task)
get_task_struct(task);
spin_unlock_irqrestore(&donor_lock, flags);
if (!task)
return -ENOENT;
if (READ_ONCE(task->__state) != TASK_RUNNING)
state |= ENQ_BLOCKED_DONOR_SLEEPING;
if (READ_ONCE(task->on_rq))
state |= ENQ_BLOCKED_DONOR_ON_RQ;
put_task_struct(task);
return state;
}
static void reset_stats(void)
{
atomic64_set(&hold_time_ns, 0);
atomic64_set(&wait_time_ns, 0);
atomic64_set(&nr_holds, 0);
atomic64_set(&nr_waits, 0);
}
static long get_stats(unsigned long arg)
{
struct enq_blocked_stats stats = {
.hold_time_ns = atomic64_read(&hold_time_ns),
.wait_time_ns = atomic64_read(&wait_time_ns),
.nr_holds = atomic64_read(&nr_holds),
.nr_waits = atomic64_read(&nr_waits),
};
return copy_to_user((void __user *)arg, &stats, sizeof(stats)) ?
-EFAULT : 0;
}
static long enq_blocked_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
switch (cmd) {
case ENQ_BLOCKED_IOCTL_OWNER:
return run_owner();
case ENQ_BLOCKED_IOCTL_DONOR:
return run_donor();
case ENQ_BLOCKED_IOCTL_RESET_STATS:
reset_stats();
return 0;
case ENQ_BLOCKED_IOCTL_GET_STATS:
return get_stats(arg);
case ENQ_BLOCKED_IOCTL_PREP_ATTACH:
atomic_set(&attach_done, 0);
atomic_set(&attach_pending, 1);
return 0;
case ENQ_BLOCKED_IOCTL_ATTACH_DONE:
atomic_set(&attach_done, 1);
return 0;
case ENQ_BLOCKED_IOCTL_DONOR_STATE:
return get_donor_state();
case ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED:
return IS_ENABLED(CONFIG_SCHED_PROXY_EXEC);
default:
return -EINVAL;
}
}
static const struct file_operations enq_blocked_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = enq_blocked_ioctl,
};
static struct miscdevice enq_blocked_device = {
.minor = MISC_DYNAMIC_MINOR,
.name = "scx_enq_blocked",
.fops = &enq_blocked_fops,
.mode = 0600,
};
module_misc_device(enq_blocked_device);
MODULE_AUTHOR("Andrea Righi <arighi@nvidia.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("sched_ext blocked donor test module");
tools/testing/selftests/sched_ext/util.c
+143
View File
@@ -5,10 +5,21 @@
*/
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>
#include <zlib.h>
#include "util.h"
#ifndef SCHED_EXT
#define SCHED_EXT 7
#endif
/* Returns read len on success, or -errno on failure. */
static ssize_t read_text(const char *path, char *buf, size_t max_len)
@@ -69,3 +80,135 @@ int file_write_long(const char *path, long val)
return 0;
}
struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext)
{
struct scx_test_gated_worker worker = { .pid = -1, .start_fd = -1 };
int ready[2], start[2];
pid_t parent = getpid();
char byte = 1;
if (pipe(ready))
return worker;
if (pipe(start)) {
close(ready[0]);
close(ready[1]);
return worker;
}
worker.pid = fork();
if (!worker.pid) {
struct sched_param param = {};
cpu_set_t mask;
close(ready[0]);
close(start[1]);
if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent)
_exit(1);
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask))
_exit(1);
if (set_sched_ext && sched_setscheduler(0, SCHED_EXT, &param))
_exit(1);
if (write(ready[1], &byte, 1) != 1)
_exit(1);
close(ready[1]);
if (read(start[0], &byte, 1) != 1)
_exit(1);
close(start[0]);
for (;;)
asm volatile("" ::: "memory");
}
if (worker.pid < 0) {
close(ready[0]);
close(ready[1]);
close(start[0]);
close(start[1]);
return worker;
}
close(ready[1]);
close(start[0]);
if (read(ready[0], &byte, 1) != 1) {
close(ready[0]);
close(start[1]);
kill(worker.pid, SIGKILL);
waitpid(worker.pid, NULL, 0);
worker.pid = -1;
return worker;
}
close(ready[0]);
worker.start_fd = start[1];
return worker;
}
bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker)
{
char byte = 1;
if (write(worker->start_fd, &byte, 1) != 1)
return false;
close(worker->start_fd);
worker->start_fd = -1;
return true;
}
void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker)
{
if (worker->start_fd >= 0)
close(worker->start_fd);
if (worker->pid > 0) {
kill(worker->pid, SIGKILL);
waitpid(worker->pid, NULL, 0);
}
worker->pid = -1;
worker->start_fd = -1;
}
static int config_preempt_lazy_mode(void)
{
bool dynamic = false, lazy = false, immediate = false;
char buf[128];
gzFile file;
file = gzopen("/proc/config.gz", "r");
if (!file)
return -1;
while (gzgets(file, buf, sizeof(buf))) {
if (!strcmp(buf, "CONFIG_PREEMPT_DYNAMIC=y\n"))
dynamic = true;
else if (!strcmp(buf, "CONFIG_PREEMPT_LAZY=y\n"))
lazy = true;
else if (!strcmp(buf, "CONFIG_PREEMPT_NONE=y\n") ||
!strcmp(buf, "CONFIG_PREEMPT_VOLUNTARY=y\n") ||
!strcmp(buf, "CONFIG_PREEMPT=y\n") ||
!strcmp(buf, "CONFIG_PREEMPT_RT=y\n"))
immediate = true;
}
gzclose(file);
if (dynamic)
return -1;
if (lazy)
return 1;
if (immediate)
return 0;
return -1;
}
int scx_test_preempt_lazy_mode(void)
{
char buf[128];
if (read_text("/sys/kernel/debug/sched/preempt", buf, sizeof(buf)) > 0) {
if (strstr(buf, "(lazy)"))
return 1;
if (strstr(buf, "(none)") || strstr(buf, "(voluntary)") ||
strstr(buf, "(full)"))
return 0;
}
return config_preempt_lazy_mode();
}
tools/testing/selftests/sched_ext/util.h
+12
View File
@@ -7,7 +7,19 @@
#ifndef __SCX_TEST_UTIL_H__
#define __SCX_TEST_UTIL_H__
#include <stdbool.h>
#include <sys/types.h>
struct scx_test_gated_worker {
pid_t pid;
int start_fd;
};
long file_read_long(const char *path);
int file_write_long(const char *path, long val);
struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext);
bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker);
void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker);
int scx_test_preempt_lazy_mode(void);
#endif // __SCX_TEST_UTIL_H__