mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-10-26 10:57:38 +08:00
Josef Bacik
8257f6ec45
Since local task_work moved to the mpscq, ->cq_wait_nr is a credit: the
add that counts it down to zero, or exchanges it for IO_CQ_WAKE_INIT,
issues the only wake up of the wait cycle, and every later add returns
early as the count is <= 0.
An add looks at the count after it has queued its entry, so an add that
gets delayed in between can take the credit of a later cycle. That was
assumed to cause an early wake up at worst, but the waiter publishes the
count before it changes its task state:
CPU0 (submitter, io_cqring_wait) CPU1 (io_req_local_work_add)
mpscq_push(req A)
io_run_local_work() // runs req A
atomic_set(&ctx->cq_wait_nr, 1)
atomic_read(&ctx->cq_wait_nr) == 1
atomic_xchg(.., IO_CQ_WAKE_INIT)
wake_up_state(.., TASK_INTERRUPTIBLE)
// still TASK_RUNNING, no-op
set_current_state(TASK_INTERRUPTIBLE)
io_local_work_pending() == false
schedule()
mpscq_push(req B)
cq_wait_nr <= 0, return
The task now sleeps with req B queued, until a signal or a timeout ends
the wait.
Set the task state first and only then make the count visible, so that
whoever takes the credit finds a task it can wake. The full barrier that
orders the count against the check of the work list is kept. Do the same
for the wait of the BPF driven loop, which copied the sequence.
Fixes: d46ab2c98a ("io_uring: switch local task_work to a mpscq")
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Link: https://patch.msgid.link/20261015-b4-io-uring-fixes-v1-7-c1556f5d7bf5@toxicpanda.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
71 lines
2.1 KiB
C
71 lines
2.1 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#ifndef IOU_WAIT_H
|
|
#define IOU_WAIT_H
|
|
|
|
#include <linux/io_uring_types.h>
|
|
|
|
/*
|
|
* ->cq_wait_nr is armed with the number of lazy task_work adds the waiter
|
|
* still needs, and counted down by the add side, with the add reaching zero
|
|
* issuing the (single) wake up for this wait cycle. Zero and below means no
|
|
* wake up is to be issued: IO_CQ_WAKE_INIT when no task is waiting (also
|
|
* what a forced wake up resets it to when claiming one), zero once the
|
|
* countdown has fired.
|
|
*/
|
|
#define IO_CQ_WAKE_INIT (-1)
|
|
|
|
/*
|
|
* Start a wait cycle of the current task. The add that uses up ->cq_wait_nr
|
|
* may be one that queued its entry long ago and only now gets to look at the
|
|
* count. As nobody else wakes us after that, it has to find us sleeping, hence
|
|
* the task state must be visible before the count is. The barrier pairs with
|
|
* the xchg() in mpscq_push(), see io_req_local_work_add().
|
|
*/
|
|
static inline void io_cq_wait_arm(struct io_ring_ctx *ctx, int nr_wait)
|
|
{
|
|
__set_current_state(TASK_INTERRUPTIBLE);
|
|
atomic_set_release(&ctx->cq_wait_nr, nr_wait);
|
|
/* order the above against looking at the work list */
|
|
smp_mb();
|
|
}
|
|
|
|
struct ext_arg {
|
|
size_t argsz;
|
|
struct timespec64 ts;
|
|
const sigset_t __user *sig;
|
|
ktime_t min_time;
|
|
bool ts_set;
|
|
bool iowait;
|
|
};
|
|
|
|
int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
|
|
struct ext_arg *ext_arg);
|
|
int io_run_task_work_sig(struct io_ring_ctx *ctx);
|
|
void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx);
|
|
void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx);
|
|
|
|
static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
|
|
{
|
|
struct io_rings *rings = io_get_rings(ctx);
|
|
return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
|
|
}
|
|
|
|
static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
|
|
{
|
|
struct io_rings *rings = io_get_rings(ctx);
|
|
|
|
return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
|
|
}
|
|
|
|
/*
|
|
* Reads the tail/head of the CQ ring while providing an acquire ordering,
|
|
* see comment at top of io_uring.c.
|
|
*/
|
|
static inline unsigned io_cqring_events(struct io_ring_ctx *ctx)
|
|
{
|
|
smp_rmb();
|
|
return __io_cqring_events(ctx);
|
|
}
|
|
|
|
#endif
|