mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-10-26 10:57:38 +08:00
Josef Bacik
8257f6ec45
Since local task_work moved to the mpscq, ->cq_wait_nr is a credit: the
add that counts it down to zero, or exchanges it for IO_CQ_WAKE_INIT,
issues the only wake up of the wait cycle, and every later add returns
early as the count is <= 0.
An add looks at the count after it has queued its entry, so an add that
gets delayed in between can take the credit of a later cycle. That was
assumed to cause an early wake up at worst, but the waiter publishes the
count before it changes its task state:
CPU0 (submitter, io_cqring_wait) CPU1 (io_req_local_work_add)
mpscq_push(req A)
io_run_local_work() // runs req A
atomic_set(&ctx->cq_wait_nr, 1)
atomic_read(&ctx->cq_wait_nr) == 1
atomic_xchg(.., IO_CQ_WAKE_INIT)
wake_up_state(.., TASK_INTERRUPTIBLE)
// still TASK_RUNNING, no-op
set_current_state(TASK_INTERRUPTIBLE)
io_local_work_pending() == false
schedule()
mpscq_push(req B)
cq_wait_nr <= 0, return
The task now sleeps with req B queued, until a signal or a timeout ends
the wait.
Set the task state first and only then make the count visible, so that
whoever takes the credit finds a task it can wake. The full barrier that
orders the count against the check of the work list is kept. Do the same
for the wait of the BPF driven loop, which copied the sequence.
Fixes: d46ab2c98a ("io_uring: switch local task_work to a mpscq")
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Link: https://patch.msgid.link/20261015-b4-io-uring-fixes-v1-7-c1556f5d7bf5@toxicpanda.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
91 lines
1.9 KiB
C
91 lines
1.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#include "io_uring.h"
|
|
#include "wait.h"
|
|
#include "loop.h"
|
|
#include "tctx.h"
|
|
|
|
static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
|
|
const struct iou_loop_params *lp)
|
|
{
|
|
return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
|
|
}
|
|
|
|
static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
|
|
{
|
|
__set_current_state(TASK_RUNNING);
|
|
atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
|
|
}
|
|
|
|
static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
|
|
unsigned nr_wait)
|
|
{
|
|
io_cq_wait_arm(ctx, nr_wait);
|
|
|
|
if (unlikely(io_local_work_pending(ctx) ||
|
|
io_loop_nr_cqes(ctx, lp) <= 0) ||
|
|
READ_ONCE(ctx->check_cq)) {
|
|
io_loop_wait_finish(ctx);
|
|
return;
|
|
}
|
|
|
|
mutex_unlock(&ctx->uring_lock);
|
|
schedule();
|
|
io_loop_wait_finish(ctx);
|
|
mutex_lock(&ctx->uring_lock);
|
|
}
|
|
|
|
static int __io_run_loop(struct io_ring_ctx *ctx)
|
|
{
|
|
struct iou_loop_params lp = {};
|
|
|
|
while (true) {
|
|
int nr_wait, step_res;
|
|
|
|
if (unlikely(!ctx->loop_step))
|
|
return -EFAULT;
|
|
|
|
step_res = ctx->loop_step(io_loop_mangle_ctx(ctx), &lp);
|
|
if (step_res == IOU_LOOP_STOP)
|
|
break;
|
|
if (step_res != IOU_LOOP_CONTINUE)
|
|
return -EINVAL;
|
|
|
|
nr_wait = io_loop_nr_cqes(ctx, &lp);
|
|
if (nr_wait > 0)
|
|
io_loop_wait(ctx, &lp, nr_wait);
|
|
else
|
|
nr_wait = 0;
|
|
|
|
if (task_work_pending(current)) {
|
|
mutex_unlock(&ctx->uring_lock);
|
|
io_run_task_work();
|
|
mutex_lock(&ctx->uring_lock);
|
|
}
|
|
if (unlikely(task_sigpending(current)))
|
|
return -EINTR;
|
|
io_run_local_work_locked(ctx, nr_wait);
|
|
|
|
if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
|
|
io_cqring_overflow_flush_locked(ctx);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int io_run_loop(struct io_ring_ctx *ctx)
|
|
{
|
|
int ret;
|
|
|
|
ret = io_uring_add_tctx_node(ctx);
|
|
if (unlikely(ret))
|
|
return ret;
|
|
|
|
if (!io_allowed_run_tw(ctx))
|
|
return -EEXIST;
|
|
|
|
mutex_lock(&ctx->uring_lock);
|
|
ret = __io_run_loop(ctx);
|
|
mutex_unlock(&ctx->uring_lock);
|
|
return ret;
|
|
}
|