mirrors/linux
0
0 Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-10-26 10:57:38 +08:00
Code Activity

Merge tag 'for-7.4/block-20261021' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Browse Source 
Pull block updates from Jens Axboe:

 - NVMe updates via Keith:
     - Apple M4 (t8132) ANS2 NVMe support
     - nvme-multipath sysfs attributes for path_state, iopolicies, and
       fail_on_ctrl_loss, plus path state evaluation fixes
     - nvme-tcp parallelized I/O queue allocation and startup, and using
       the caller's network namespace
     - nvmet passthru UAF fix and rework, PR preempt use-after-free
       fixes, auth hardening, and traffic based keep alive support
     - DH-HMAC-CHAP secret attributes readable by root only
     - Various other fixes and cleanups

 - swim driver overhaul and fixes, plus floppy fixes, via a pull from
   Denis

 - Block device buffered and direct I/O rework from Tal: use
   iomap_dirty_folio, take i_rwsem for the write fallback and splice
   read, honor IOCB_NOWAIT in the buffered read path, and reject atomic
   write fallback to buffered I/O

 - Large zone revalidation rework: serialize revalidation, handle
   capacity changes by dropping and retrying zone write plugs, propagate
   read-only and offline conditions, and treat offline and read-only
   zones as dead

 - ublk robustness work around canceled FETCH rounds and going live over
   canceled io commands, dropping the default I/O scheduler, and a batch
   of selftest additions

 - blk-iocost flush cost support and zone append accounting

 - rust block fixes for Send/Sync bounds on GenDisk and TagSet, and
   import style cleanups

 - Atomic write fixes capping size by the PI buffer constraints and
   failing short atomic pins

 - Skip the redundant flush for O_DSYNC direct writes, and only use
   REQ_FUA when the device supports it

 - Store GPT attributes as a raw value

 - Remove support for swapping loop fds

 - Various other fixes and cleanups all over

* tag 'for-7.4/block-20261021' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (144 commits)
  block: restore the iterator when rejecting a misaligned bio
  blk-mq: clear stale inactive flag on hctx reuse
  block, bfq: initialize active lists for all actuators
  zloop: print device that had errors
  ublk: pin module while release work is pending
  blk-mq: drop the tag_set argument from blk_mq_free_rq_map/tags
  blk-mq: share one tags_srcu instance across all tag sets
  blk-iocost: fix stale comment in ioc_rqos_throttle()
  blk-iocost: account zone append completions in latency stats
  blk-iocost: charge zone appends as writes
  blk-iocost: add flush cost support with the flushiops model parameter
  ublk: use global index for reported zones
  selftests: ublk: add test for going live over canceled io commands
  selftests: ublk: stop a batch thread when PREP_IO_CMDS fails
  selftests: ublk: move the control command helpers into ctrl.c
  ublk: refuse to go live after an io command was canceled
  ublk: let STOP_DEV cancel the server's commands before its release
  ublk: allow a NULL io->cmd in the cancel callback
  ublk: reset the FETCH round under ub->mutex
  ublk: reset the FETCH round in release also without a disk
  ...
This commit is contained in:
Linus Torvalds committed 2026-10-24 14:14:00 -10:00
2 parents fa899d500a 40d070cbb1
commit b23545529c
66 files changed
+3788 -1536
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/ABI/stable/sysfs-nvme
+52
View File
@@ -326,6 +326,27 @@ Description:
currently selected path. Returns empty if iopolicy is not
"numa". Requires CONFIG_NVME_MULTIPATH.
What: /sys/block/nvmeXcYnZ/path_state
Date: September 2026
KernelVersion: 7.4
Contact: Guixin Liu <kanie@linux.alibaba.com>
Description:
Shows the current multipath path state for this namespace
path device. Possible values:
"enabled": the path is usable for I/O.
"ctrl-down": the path is disabled because the controller
is not live.
"ana-pending": the path is disabled because an ANA state
change is pending.
"ns-not-ready": the path is disabled because the namespace
is not ready.
Requires CONFIG_NVME_MULTIPATH.
What: /sys/block/nvmeXnY/delayed_removal_secs
Date: May 2025
KernelVersion: 6.16
@@ -337,6 +358,22 @@ Description:
is deferred. Only visible on multipath head devices.
Requires CONFIG_NVME_MULTIPATH.
What: /sys/block/nvmeXnY/fail_on_ctrl_loss
Date: October 2026
KernelVersion: 7.4
Contact: Krishna Iyer <kiyer@crusoe.ai>
Description:
Shows or sets the fail-on-ctrl-loss policy of the multipath
head device ("on" or "off", default "off"). When on, I/O
queued or arriving while no usable path exists is failed
immediately instead of being queued. This is the opposite of
the queue-if-no-path behavior enabled by a nonzero
delayed_removal_secs: when both are set fail_on_ctrl_loss takes
precedence and the parked I/O is failed rather than held for
the delayed_removal_secs window. Reconnect attempts are not
affected. Only visible on multipath head devices.
Requires CONFIG_NVME_MULTIPATH.
What: /sys/block/nvmeXnY/csi
What: /sys/block/nvmeXnY/metadata_bytes
What: /sys/block/nvmeXnY/nuse
@@ -444,6 +481,21 @@ Description:
selections. Only available when CONFIG_NVME_MULTIPATH is
enabled.
A discovery subsystem has no namespaces and hence no I/O
paths to select from: reading returns "none" and writing
fails with EOPNOTSUPP.
What: /sys/class/nvme-subsystem/nvme-subsysX/iopolicies
Date: September 2026
KernelVersion: 7.4
Contact: Laurence Oberman <loberman@redhat.com>
Description:
Shows the I/O path selection policies that may be written
to the iopolicy attribute of this subsystem, separated by
spaces: "numa round-robin queue-depth". A discovery
subsystem accepts no policy at all and reports "none".
Only available when CONFIG_NVME_MULTIPATH is enabled.
What: /sys/class/nvme-subsystem/nvme-subsysX/subsystype
Date: September 2021
KernelVersion: 5.16
Documentation/admin-guide/cgroup-v2.rst
+7
View File
@@ -2131,6 +2131,7 @@ IO Interface Files
[r|w]bps The maximum sequential IO throughput
[r|w]seqiops The maximum 4k sequential IOs per second
[r|w]randiops The maximum 4k random IOs per second
flushiops The maximum flushes per second
============= ========================================
From the above, the builtin linear model determines the base
@@ -2138,6 +2139,12 @@ IO Interface Files
for the IO size. While simple, this model can cover most
common device classes acceptably.
"flushiops" determines the cost of a cache flush: a write with
a preceding cache flush is charged one flush on top of its data
cost, and a FUA write one more flush on devices without native
FUA support. It is zero in the builtin profiles, so flushes
stay free until it is configured.
The IO cost model isn't expected to be accurate in absolute
sense and is scaled to the device behavior dynamically.
Documentation/block/ublk.rst
+15 -2
View File
@@ -118,7 +118,11 @@ managing and controlling ublk devices with help of several control commands:
After the server prepares userspace resources (such as creating I/O handler
threads & io_uring for handling ublk IO), this command is sent to the
driver for allocating & exposing ``/dev/ublkb*``. Parameters set via
``UBLK_CMD_SET_PARAMS`` are applied for creating the device.
``UBLK_CMD_SET_PARAMS`` are applied for creating the device. The command
fails with ``-EBUSY`` if an I/O command fetched by the current server
was canceled, because its io_uring is gone. The server can't fetch it
again, and the device can be started again once the server has closed
``/dev/ublkc*``.
- ``UBLK_CMD_STOP_DEV``
@@ -126,6 +130,13 @@ managing and controlling ublk devices with help of several control commands:
ublk server will release resources (such as destroying I/O handler threads &
io_uring).
If a server has ``/dev/ublkc*`` open, its fetched I/O commands complete with
``UBLK_IO_RES_ABORT``. Until it closes ``/dev/ublkc*``, new
``UBLK_IO_FETCH_REQ`` and ``UBLK_U_IO_PREP_IO_CMDS`` commands complete with
``UBLK_IO_RES_ABORT`` too, and ``UBLK_CMD_START_DEV`` fails with ``-EBUSY``,
even if the server hadn't fetched anything. The device can be started again
by a server that opens ``/dev/ublkc*`` after that.
- ``UBLK_CMD_DEL_DEV``
Remove ``/dev/ublkc*``. When this command returns, the allocated ublk device
@@ -195,7 +206,9 @@ managing and controlling ublk devices with help of several control commands:
command is accepted after ublk device is quiesced and a new process has
opened ``/dev/ublkc*`` and get all ublk queues be ready. When this command
returns, ublk device is unquiesced and new I/O requests are passed to the
new process.
new process. It fails with ``-EBUSY`` if an I/O command of the new
process was canceled already. The recovery can be started over once the
new process has closed ``/dev/ublkc*``.
- user recovery feature description
Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml
+1
View File
@@ -16,6 +16,7 @@ properties:
- items:
- const: apple,t6020-nvme-ans2
- const: apple,t8103-nvme-ans2
- const: apple,t8132-nvme-ans2
- items:
- enum:
# Do not add additional SoC to this list.
arch/m68k/mac/config.c
+30 -1
View File
@@ -984,8 +984,37 @@ static int __init mac_platform_init(void)
.start = swim_base,
.end = swim_base + 0x1FFF,
};
bool fast_fclk = false;
platform_device_register_simple("swim", -1, &swim_rsrc, 1);
/* Models in this list are supposed to run FCLK at 32 MHz */
switch (macintosh_config->ident) {
case MAC_MODEL_CCL:
case MAC_MODEL_CCLII:
case MAC_MODEL_LCIII:
case MAC_MODEL_Q605:
case MAC_MODEL_Q605_ACC:
case MAC_MODEL_Q610:
case MAC_MODEL_Q630:
case MAC_MODEL_Q650:
case MAC_MODEL_Q800:
case MAC_MODEL_P460:
case MAC_MODEL_P475:
case MAC_MODEL_P475F:
case MAC_MODEL_P520:
case MAC_MODEL_P550:
case MAC_MODEL_P575:
case MAC_MODEL_P588:
case MAC_MODEL_TV:
case MAC_MODEL_C610:
case MAC_MODEL_C650:
case MAC_MODEL_PB190:
case MAC_MODEL_PB520:
fast_fclk = true;
break;
}
platform_device_register_resndata(NULL, "swim", -1, &swim_rsrc, 1,
&fast_fclk, sizeof(fast_fclk));
}
/*
block/bdev.c
+1 -1
View File
@@ -292,7 +292,7 @@ EXPORT_SYMBOL(sync_blockdev_range);
* The reference counter (bd_fsfreeze_count) guarantees that only the last
* unfreeze process can unfreeze the frozen filesystem actually when multiple
* freeze requests arrive simultaneously. It counts up in bdev_freeze() and
* count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
* count down in bdev_thaw(). When it becomes 0, bdev_thaw() will unfreeze
* actually.
*
* Return: On success zero is returned, negative error code on failure.
block/bfq-iosched.c
+2 -2
View File
@@ -7273,8 +7273,8 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq)
bfqd->num_groups_with_pending_reqs = 0;
#endif
INIT_LIST_HEAD(&bfqd->active_list[0]);
INIT_LIST_HEAD(&bfqd->active_list[1]);
for (i = 0; i < bfqd->num_actuators; i++)
INIT_LIST_HEAD(&bfqd->active_list[i]);
INIT_LIST_HEAD(&bfqd->idle_list);
INIT_HLIST_HEAD(&bfqd->burst_list);
block/bio.c
+60 -17
View File
@@ -1217,6 +1217,21 @@ bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter)
return true;
}
static unsigned int bvec_nr_pages(const struct bio_vec *bv)
{
return (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE -
bv->bv_offset / PAGE_SIZE + 1;
}
static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
{
struct folio *folio = bvec_folio(bv);
if (mark_dirty)
folio_mark_dirty_lock(folio);
unpin_user_folio(folio, bvec_nr_pages(bv));
}
/*
* Aligns the bio size to the len_align_mask, releasing excessive bio vecs that
* __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length
@@ -1227,6 +1242,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
{
struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
unsigned int npages;
if (!nbytes)
return 0;
@@ -1235,14 +1251,24 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
bio->bi_iter.bi_size -= nbytes;
while (nbytes >= bv->bv_len) {
if (bio_flagged(bio, BIO_PAGE_PINNED))
unpin_user_page(bv->bv_page);
bvec_unpin(bv, false);
if (!--bio->bi_vcnt)
return -EFAULT;
nbytes -= bv->bv_len;
bv--;
}
/*
* __bio_release_pages() only unpins the pages still covered by
* the trimmed bv_len. Count the pages spanned before and after
* the trim and unpin the difference.
*/
npages = bvec_nr_pages(bv);
bv->bv_len -= nbytes;
npages -= bvec_nr_pages(bv);
if (npages && bio_flagged(bio, BIO_PAGE_PINNED))
unpin_user_folio(bvec_folio(bv), npages);
return 0;
}
@@ -1308,6 +1334,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
unsigned len_align_mask)
{
iov_iter_extraction_t flags = 0;
struct iov_iter saved_iter;
int ret;
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
return -EIO;
@@ -1328,34 +1356,49 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue))
flags |= ITER_ALLOW_P2PDMA;
saved_iter = *iter;
do {
ssize_t ret;
ssize_t size;
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
size = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
maxlen - bio->bi_iter.bi_size,
&bio->bi_vcnt, bio->bi_max_vecs,
mem_align_mask, flags);
if (ret <= 0) {
/*
* A misaligned vector fails the whole I/O. Release any
* pages pinned by earlier iterations before returning
* since this bio won't be submitted to release them.
*/
if (ret == -EINVAL) {
bio_release_pages(bio, false);
bio_clear_flag(bio, BIO_PAGE_PINNED);
bio->bi_vcnt = 0;
}
if (size <= 0) {
/* A misaligned vector fails the whole I/O */
if (size == -EINVAL)
goto out_release_pages;
if (!bio->bi_vcnt)
return ret;
return size;
break;
}
bio->bi_iter.bi_size += ret;
bio->bi_iter.bi_size += size;
} while (iov_iter_count(iter) && !bio_full(bio, 0));
if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
bio->bi_opf |= REQ_NOMERGE;
return bio_iov_iter_align_down(bio, iter, len_align_mask);
ret = bio_iov_iter_align_down(bio, iter, len_align_mask);
if (ret)
return ret;
/*
* An atomic write is submitted as a single bio, so it has to cover
* the whole iterator or it would be torn.
*/
if ((bio->bi_opf & REQ_ATOMIC) && iov_iter_count(iter))
goto out_release_pages;
return 0;
out_release_pages:
/*
* Release the pages pinned so far before failing, since this bio won't
* be submitted to release them.
*/
bio_release_pages(bio, false);
bio_clear_flag(bio, BIO_PAGE_PINNED);
bio->bi_vcnt = 0;
*iter = saved_iter;
return -EINVAL;
}
static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
block/blk-core.c
+5 -2
View File
@@ -902,8 +902,11 @@ void submit_bio_noacct(struct bio *bio)
case REQ_OP_ZONE_CLOSE:
case REQ_OP_ZONE_RESET:
case REQ_OP_ZONE_FINISH:
/* Zone management operations require sequential zones. */
if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector))
/*
* Zone management operations require sequential zones that are
* not offline nor read-only.
*/
if (!bdev_zone_mgmt_allowed(bdev, bio->bi_iter.bi_sector))
goto end_io;
break;
case REQ_OP_ZONE_RESET_ALL:
block/blk-iocost.c
+31 -8
View File
@@ -353,6 +353,7 @@ enum {
I_LCOEF_WBPS,
I_LCOEF_WSEQIOPS,
I_LCOEF_WRANDIOPS,
I_LCOEF_FLUSHIOPS,
NR_I_LCOEFS,
};
@@ -363,6 +364,7 @@ enum {
LCOEF_WPAGE,
LCOEF_WSEQIO,
LCOEF_WRANDIO,
LCOEF_FLUSH,
NR_LCOEFS,
};
@@ -883,6 +885,9 @@ static void ioc_refresh_lcoefs(struct ioc *ioc)
&c[LCOEF_RPAGE], &c[LCOEF_RSEQIO], &c[LCOEF_RRANDIO]);
calc_lcoefs(u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS],
&c[LCOEF_WPAGE], &c[LCOEF_WSEQIO], &c[LCOEF_WRANDIO]);
c[LCOEF_FLUSH] = u[I_LCOEF_FLUSHIOPS] ?
DIV64_U64_ROUND_UP(VTIME_PER_SEC, u[I_LCOEF_FLUSHIOPS]) : 0;
}
/*
@@ -2533,7 +2538,16 @@ static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
u64 seek_pages = 0;
u64 cost = 0;
/* Can't calculate cost for empty bio */
/*
* FUA on a device without native support becomes a post-flush;
* charge it like PREFLUSH from the flush coefficient.
*/
if (bio->bi_opf & REQ_PREFLUSH)
cost += ioc->params.lcoefs[LCOEF_FLUSH];
if ((bio->bi_opf & REQ_FUA) && !bdev_fua(bio->bi_bdev))
cost += ioc->params.lcoefs[LCOEF_FLUSH];
/* Can't calculate data cost for empty bio */
if (!bio->bi_iter.bi_size)
goto out;
@@ -2543,6 +2557,7 @@ static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
coef_randio = ioc->params.lcoefs[LCOEF_RRANDIO];
coef_page = ioc->params.lcoefs[LCOEF_RPAGE];
break;
case REQ_OP_ZONE_APPEND:
case REQ_OP_WRITE:
coef_seqio = ioc->params.lcoefs[LCOEF_WSEQIO];
coef_randio = ioc->params.lcoefs[LCOEF_WRANDIO];
@@ -2586,6 +2601,7 @@ static void calc_size_vtime_cost_builtin(struct request *rq, struct ioc *ioc,
case REQ_OP_READ:
*costp = pages * ioc->params.lcoefs[LCOEF_RPAGE];
break;
case REQ_OP_ZONE_APPEND:
case REQ_OP_WRITE:
*costp = pages * ioc->params.lcoefs[LCOEF_WPAGE];
break;
@@ -2708,7 +2724,9 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
if (!iocg_activate(iocg, &now))
return;
iocg->cursor = bio_end_sector(bio);
/* dataless bios have no meaningful position for seq/rand detection */
if (bio->bi_iter.bi_size)
iocg->cursor = bio_end_sector(bio);
vtime = atomic64_read(&iocg->vtime);
cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);
@@ -2725,10 +2743,11 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
/*
* We're over budget. This can be handled in two ways. IOs which may
* cause priority inversions are punted to @ioc->aux_iocg and charged as
* debt. Otherwise, the issuer is blocked on @iocg->waitq. Debt handling
* requires @ioc->lock, waitq handling @iocg->waitq.lock. Determine
* whether debt handling is needed and acquire locks accordingly.
* cause priority inversions are issued regardless and charged against
* @iocg->abs_vdebt as debt. Otherwise, the issuer is blocked on
* @iocg->waitq. Debt handling requires @ioc->lock, waitq handling
* @iocg->waitq.lock. Determine whether debt handling is needed and
* acquire locks accordingly.
*/
use_debt = bio_issue_as_root_blkg(bio) || fatal_signal_pending(current);
ioc_locked = use_debt || READ_ONCE(iocg->abs_vdebt);
@@ -2854,6 +2873,7 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
pidx = QOS_RLAT;
rw = READ;
break;
case REQ_OP_ZONE_APPEND:
case REQ_OP_WRITE:
pidx = QOS_WLAT;
rw = WRITE;
@@ -3440,10 +3460,12 @@ static u64 ioc_cost_model_prfill(struct seq_file *sf,
spin_lock_irq(&ioc->lock);
seq_printf(sf, "%s ctrl=%s model=linear "
"rbps=%llu rseqiops=%llu rrandiops=%llu "
"wbps=%llu wseqiops=%llu wrandiops=%llu\n",
"wbps=%llu wseqiops=%llu wrandiops=%llu "
"flushiops=%llu\n",
dname, ioc->user_cost_model ? "user" : "auto",
u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS],
u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS]);
u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS],
u[I_LCOEF_FLUSHIOPS]);
spin_unlock_irq(&ioc->lock);
return 0;
}
@@ -3470,6 +3492,7 @@ static const match_table_t i_lcoef_tokens = {
{ I_LCOEF_WBPS, "wbps=%u" },
{ I_LCOEF_WSEQIOPS, "wseqiops=%u" },
{ I_LCOEF_WRANDIOPS, "wrandiops=%u" },
{ I_LCOEF_FLUSHIOPS, "flushiops=%u" },
{ NR_I_LCOEFS, NULL },
};
block/blk-mq-tag.c
+13 -6
View File
@@ -18,6 +18,13 @@
#include "blk-mq.h"
#include "blk-mq-sched.h"
/*
* Protects tag iteration against the deferred freeing of tags->page_list and
* flush queues. The callbacks touch neither the tag set nor driver code, so
* one global instance is enough and freeing a tag set needn't wait for them.
*/
DEFINE_SRCU(blk_mq_tags_srcu);
/*
* Recalculate wakeup batch when tag is shared by hctx.
*/
@@ -443,7 +450,7 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
unsigned int flags = tagset->flags;
int i, nr_tags, srcu_idx;
srcu_idx = srcu_read_lock(&tagset->tags_srcu);
srcu_idx = srcu_read_lock(&blk_mq_tags_srcu);
nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues;
@@ -452,7 +459,7 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
__blk_mq_all_tag_iter(tagset->tags[i], fn, priv,
BT_TAG_ITER_STARTED);
}
srcu_read_unlock(&tagset->tags_srcu, srcu_idx);
srcu_read_unlock(&blk_mq_tags_srcu, srcu_idx);
}
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
@@ -512,7 +519,7 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn,
if (!percpu_ref_tryget(&q->q_usage_counter))
return;
srcu_idx = srcu_read_lock(&q->tag_set->tags_srcu);
srcu_idx = srcu_read_lock(&blk_mq_tags_srcu);
if (blk_mq_is_shared_tags(q->tag_set->flags)) {
struct blk_mq_tags *tags = q->tag_set->shared_tags;
struct sbitmap_queue *bresv = &tags->breserved_tags;
@@ -542,7 +549,7 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn,
bt_for_each(hctx, q, btags, fn, priv, false);
}
}
srcu_read_unlock(&q->tag_set->tags_srcu, srcu_idx);
srcu_read_unlock(&blk_mq_tags_srcu, srcu_idx);
blk_queue_exit(q);
}
@@ -607,7 +614,7 @@ static void blk_mq_free_tags_callback(struct rcu_head *head)
kfree(tags);
}
void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags)
void blk_mq_free_tags(struct blk_mq_tags *tags)
{
sbitmap_queue_free(&tags->bitmap_tags);
sbitmap_queue_free(&tags->breserved_tags);
@@ -618,7 +625,7 @@ void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags)
return;
}
call_srcu(&set->tags_srcu, &tags->rcu_head, blk_mq_free_tags_callback);
call_srcu(&blk_mq_tags_srcu, &tags->rcu_head, blk_mq_free_tags_callback);
}
void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size)
block/blk-mq.c
+11 -16
View File
@@ -1397,6 +1397,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
(!blk_queue_nomerges(rq->q) &&
blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
blk_mq_flush_plug_list(plug, false);
blk_plug_invalidate_ts();
last = NULL;
trace_block_plug(rq->q);
}
@@ -3507,14 +3508,14 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
*/
}
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags)
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
kfree(tags->rqs);
tags->rqs = NULL;
kfree(tags->static_rqs);
tags->static_rqs = NULL;
blk_mq_free_tags(set, tags);
blk_mq_free_tags(tags);
}
static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
@@ -3576,7 +3577,7 @@ static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
err_free_rqs:
kfree(tags->rqs);
err_free_tags:
blk_mq_free_tags(set, tags);
blk_mq_free_tags(tags);
return NULL;
}
@@ -3694,9 +3695,9 @@ static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
};
int srcu_idx;
srcu_idx = srcu_read_lock(&hctx->queue->tag_set->tags_srcu);
srcu_idx = srcu_read_lock(&blk_mq_tags_srcu);
blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
srcu_read_unlock(&hctx->queue->tag_set->tags_srcu, srcu_idx);
srcu_read_unlock(&blk_mq_tags_srcu, srcu_idx);
return data.has_rq;
}
@@ -3968,7 +3969,7 @@ static void blk_mq_exit_hctx(struct request_queue *q,
if (set->ops->exit_hctx)
set->ops->exit_hctx(hctx, hctx_idx);
call_srcu(&set->tags_srcu, &hctx->fq->rcu_head,
call_srcu(&blk_mq_tags_srcu, &hctx->fq->rcu_head,
blk_free_flush_queue_callback);
hctx->fq = NULL;
@@ -4002,6 +4003,7 @@ static int blk_mq_init_hctx(struct request_queue *q,
goto fail;
hctx->queue_num = hctx_idx;
clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
hctx->tags = set->tags[hctx_idx];
@@ -4129,7 +4131,7 @@ struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
if (ret) {
blk_mq_free_rq_map(set, tags);
blk_mq_free_rq_map(tags);
return NULL;
}
@@ -4157,7 +4159,7 @@ void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
{
if (tags) {
blk_mq_free_rqs(set, tags, hctx_idx);
blk_mq_free_rq_map(set, tags);
blk_mq_free_rq_map(tags);
}
}
@@ -4895,9 +4897,6 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
if (ret)
goto out_free_srcu;
}
ret = init_srcu_struct(&set->tags_srcu);
if (ret)
goto out_cleanup_srcu;
init_rwsem(&set->update_nr_hwq_lock);
@@ -4906,7 +4905,7 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
sizeof(struct blk_mq_tags *), GFP_KERNEL,
set->numa_node);
if (!set->tags)
goto out_cleanup_tags_srcu;
goto out_cleanup_srcu;
for (i = 0; i < set->nr_maps; i++) {
set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
@@ -4935,8 +4934,6 @@ out_free_mq_map:
}
kfree(set->tags);
set->tags = NULL;
out_cleanup_tags_srcu:
cleanup_srcu_struct(&set->tags_srcu);
out_cleanup_srcu:
if (set->flags & BLK_MQ_F_BLOCKING)
cleanup_srcu_struct(set->srcu);
@@ -4983,8 +4980,6 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
kfree(set->tags);
set->tags = NULL;
srcu_barrier(&set->tags_srcu);
cleanup_srcu_struct(&set->tags_srcu);
if (set->flags & BLK_MQ_F_BLOCKING) {
srcu_barrier(set->srcu);
cleanup_srcu_struct(set->srcu);
block/blk-mq.h
+4 -2
View File
@@ -62,7 +62,7 @@ void blk_mq_put_rq_ref(struct request *rq);
*/
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx);
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags);
void blk_mq_free_rq_map(struct blk_mq_tags *tags);
struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
unsigned int hctx_idx, unsigned int depth);
void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
@@ -176,7 +176,9 @@ struct blk_mq_alloc_data {
struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,
unsigned int reserved_tags, unsigned int flags, int node);
void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags);
void blk_mq_free_tags(struct blk_mq_tags *tags);
extern struct srcu_struct blk_mq_tags_srcu;
unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags,
block/blk-settings.c
+10 -8
View File
@@ -206,12 +206,6 @@ static int blk_validate_integrity_limits(struct queue_limits *lim)
lim->max_sectors = min(lim->max_sectors,
max_integrity_io_size(lim) >> SECTOR_SHIFT);
if (lim->features & BLK_FEAT_ATOMIC_WRITES) {
lim->atomic_write_max_sectors =
min(lim->atomic_write_max_sectors,
max_integrity_io_size(lim) >> SECTOR_SHIFT);
}
return 0;
}
@@ -236,14 +230,20 @@ static unsigned int blk_queue_max_guaranteed_bio(struct queue_limits *lim)
static void blk_atomic_writes_update_limits(struct queue_limits *lim)
{
unsigned int integrity_max = lim->integrity.metadata_size ?
max_integrity_io_size(lim) : 0;
unsigned int unit_limit = min(lim->max_hw_sectors << SECTOR_SHIFT,
blk_queue_max_guaranteed_bio(lim));
unit_limit = min_not_zero(unit_limit, integrity_max);
unit_limit = rounddown_pow_of_two(unit_limit);
lim->atomic_write_max_sectors =
min(lim->atomic_write_hw_max >> SECTOR_SHIFT,
lim->max_hw_sectors);
lim->atomic_write_max_sectors =
min_not_zero(lim->atomic_write_max_sectors,
integrity_max >> SECTOR_SHIFT);
lim->atomic_write_unit_min =
min(lim->atomic_write_hw_unit_min, unit_limit);
lim->atomic_write_unit_max =
@@ -511,11 +511,13 @@ int blk_validate_limits(struct queue_limits *lim)
if (!(lim->features & BLK_FEAT_WRITE_CACHE))
lim->features &= ~BLK_FEAT_FUA;
blk_validate_atomic_write_limits(lim);
err = blk_validate_integrity_limits(lim);
if (err)
return err;
/* atomics limits depend on integrity limits */
blk_validate_atomic_write_limits(lim);
return blk_validate_zoned_limits(lim);
}
EXPORT_SYMBOL_GPL(blk_validate_limits);
block/blk-zoned.c
+550 -330
View File
File diff suppressed because it is too large. Load diff
block/blk.h
+6
View File
@@ -577,6 +577,7 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
unsigned long arg);
int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
unsigned int cmd, unsigned long arg);
bool bdev_zone_mgmt_allowed(struct block_device *bdev, sector_t sector);
#else /* CONFIG_BLK_DEV_ZONED */
static inline void disk_init_zone_resources(struct gendisk *disk)
{
@@ -619,6 +620,11 @@ static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
{
return -ENOTTY;
}
static inline bool bdev_zone_mgmt_allowed(struct block_device *bdev,
sector_t sector)
{
return false;
}
#endif /* CONFIG_BLK_DEV_ZONED */
struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
block/fops.c
+104 -37
View File
@@ -26,12 +26,26 @@ static inline struct inode *bdev_file_inode(struct file *file)
return file->f_mapping->host;
}
static blk_opf_t dio_bio_write_op(struct kiocb *iocb)
static bool blkdev_dio_fua(struct kiocb *iocb, struct block_device *bdev)
{
if (!iocb_is_dsync(iocb))
return false;
/*
* Async writes cannot fall back to generic_write_sync(), so they must
* use FUA (emulated if needed); sync writes only need it when the
* device supports FUA natively.
*/
if (!is_sync_kiocb(iocb))
return true;
return bdev_fua(bdev);
}
static blk_opf_t dio_bio_write_op(struct kiocb *iocb, struct block_device *bdev)
{
blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
/* avoid the need for a I/O completion work item */
if (iocb_is_dsync(iocb))
if (blkdev_dio_fua(iocb, bdev))
opf |= REQ_FUA;
return opf;
}
@@ -76,7 +90,8 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
if (user_backed_iter(iter))
should_dirty = true;
} else {
bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));
bio_init(&bio, bdev, vecs, nr_pages,
dio_bio_write_op(iocb, bdev));
}
bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;
bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;
@@ -180,7 +195,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
struct blkdev_dio *dio;
struct bio *bio;
bool is_read = (iov_iter_rw(iter) == READ), is_sync;
blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
loff_t pos = iocb->ki_pos;
int ret = 0;
@@ -307,9 +322,6 @@ static void blkdev_bio_end_io_async(struct bio *bio)
ret = blk_status_to_errno(bio->bi_status);
}
if (bio_integrity(bio))
bio_integrity_unmap_user(bio);
iocb->ki_complete(iocb, ret);
if (dio->flags & DIO_SHOULD_DIRTY) {
@@ -326,7 +338,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
unsigned int nr_pages)
{
bool is_read = iov_iter_rw(iter) == READ;
blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
struct blkdev_dio *dio;
struct bio *bio;
loff_t pos = iocb->ki_pos;
@@ -343,6 +355,12 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
bio->bi_end_io = blkdev_bio_end_io_async;
bio->bi_ioprio = iocb->ki_ioprio;
if (iocb->ki_flags & IOCB_ATOMIC)
bio->bi_opf |= REQ_ATOMIC;
if (iocb->ki_flags & IOCB_NOWAIT)
bio->bi_opf |= REQ_NOWAIT;
/*
* Users don't rely on the iterator being in any particular
* state for async I/O returning -EIOCBQUEUED, hence we can
@@ -365,19 +383,6 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
task_io_account_write(bio->bi_iter.bi_size);
}
if (iocb->ki_flags & IOCB_HAS_METADATA) {
ret = bio_integrity_map_iter(bio, iocb->private);
WRITE_ONCE(iocb->private, NULL);
if (unlikely(ret))
goto out_bio_put;
}
if (iocb->ki_flags & IOCB_ATOMIC)
bio->bi_opf |= REQ_ATOMIC;
if (iocb->ki_flags & IOCB_NOWAIT)
bio->bi_opf |= REQ_NOWAIT;
if (iocb->ki_flags & IOCB_HIPRI) {
bio->bi_opf |= REQ_POLLED;
submit_bio(bio);
@@ -566,7 +571,7 @@ static int blkdev_writepages(struct address_space *mapping,
}
const struct address_space_operations def_blk_aops = {
.dirty_folio = filemap_dirty_folio,
.dirty_folio = iomap_dirty_folio,
.release_folio = iomap_release_folio,
.invalidate_folio = iomap_invalidate_folio,
.read_folio = blkdev_read_folio,
@@ -715,8 +720,22 @@ blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from)
static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)
{
return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL,
NULL);
struct inode *inode = bdev_file_inode(iocb->ki_filp);
ssize_t ret;
/* Buffered fallback waits for writeback, which NOWAIT cannot allow. */
if (iocb->ki_flags & IOCB_NOWAIT)
return -EAGAIN;
/*
* Take i_rwsem to avoid racing with set_blocksize() changing
* i_blkbits/folio order and truncating the page cache.
*/
inode_lock_shared(inode);
ret = iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL,
NULL);
inode_unlock_shared(inode);
return ret;
}
/*
@@ -734,6 +753,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
bool atomic = iocb->ki_flags & IOCB_ATOMIC;
loff_t size = bdev_nr_bytes(bdev);
size_t shorted = 0;
bool need_sync = false;
ssize_t ret;
if (bdev_read_only(bdev))
@@ -771,21 +791,44 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
if (iocb->ki_flags & IOCB_DIRECT) {
ret = blkdev_direct_write(iocb, from);
if (ret >= 0 && iov_iter_count(from))
ret = direct_write_fallback(iocb, from, ret,
blkdev_buffered_write(iocb, from));
if (ret >= 0 && iov_iter_count(from)) {
if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_ATOMIC)) {
/*
* The buffered fallback blocks on i_rwsem and
* on writeback of the data it copied, and
* can't provide torn-write protection: return
* the short direct write instead and let the
* caller retry. The bytes already written
* still need a flush if REQ_FUA was not set.
*/
if (!ret)
ret = -EAGAIN;
else if (!blkdev_dio_fua(iocb, bdev))
need_sync = true;
} else {
ssize_t ret2;
inode_lock_shared(bd_inode);
ret2 = blkdev_buffered_write(iocb, from);
inode_unlock_shared(bd_inode);
ret = direct_write_fallback(iocb, from, ret,
ret2);
need_sync = true;
}
} else if (ret > 0 && !blkdev_dio_fua(iocb, bdev)) {
/*
* The device does not support FUA, so REQ_FUA was not
* set and the O_DSYNC direct write still needs a flush
* to be made durable.
*/
need_sync = true;
}
} else {
/*
* Take i_rwsem and invalidate_lock to avoid racing with
* set_blocksize changing i_blkbits/folio order and punching
* out the pagecache.
*/
inode_lock_shared(bd_inode);
ret = blkdev_buffered_write(iocb, from);
inode_unlock_shared(bd_inode);
need_sync = true;
}
if (ret > 0)
if (ret > 0 && need_sync)
ret = generic_write_sync(iocb, ret);
iov_iter_reexpand(from, iov_iter_count(from) + shorted);
return ret;
@@ -834,7 +877,15 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
* Take i_rwsem and invalidate_lock to avoid racing with set_blocksize
* changing i_blkbits/folio order and punching out the pagecache.
*/
inode_lock_shared(bd_inode);
if (iocb->ki_flags & IOCB_NOWAIT) {
if (!inode_trylock_shared(bd_inode)) {
if (!ret)
ret = -EAGAIN;
goto reexpand;
}
} else {
inode_lock_shared(bd_inode);
}
ret = filemap_read(iocb, to, ret);
inode_unlock_shared(bd_inode);
@@ -844,6 +895,22 @@ reexpand:
return ret;
}
/*
* Take i_rwsem to avoid racing with set_blocksize changing i_blkbits/folio
* order and punching out the pagecache.
*/
static ssize_t blkdev_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe, size_t len, unsigned int flags)
{
struct inode *bd_inode = bdev_file_inode(in);
ssize_t ret;
inode_lock_shared(bd_inode);
ret = filemap_splice_read(in, ppos, pipe, len, flags);
inode_unlock_shared(bd_inode);
return ret;
}
#define BLKDEV_FALLOC_FL_SUPPORTED \
(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
FALLOC_FL_ZERO_RANGE | FALLOC_FL_WRITE_ZEROES)
@@ -945,7 +1012,7 @@ const struct file_operations def_blk_fops = {
#ifdef CONFIG_COMPAT
.compat_ioctl = compat_blkdev_ioctl,
#endif
.splice_read = filemap_splice_read,
.splice_read = blkdev_splice_read,
.splice_write = iter_file_splice_write,
.fallocate = blkdev_fallocate,
.uring_cmd = blkdev_uring_cmd,
drivers/block/amiflop.c
+63 -8
View File
@@ -265,6 +265,8 @@ static struct fd_data_type data_types[] = {
static struct amiga_floppy_struct unit[FD_MAX_UNITS];
static struct timer_list flush_track_timer[FD_MAX_UNITS];
static DEFINE_SPINLOCK(flush_track_timer_lock);
static bool flush_track_timer_stopped[FD_MAX_UNITS];
static struct timer_list post_write_timer;
static unsigned long post_write_timer_drive;
static struct timer_list motor_on_timer;
@@ -1370,11 +1372,22 @@ static void flush_track_callback(struct timer_list *timer)
sizeof(flush_track_timer[0]);
nr&=3;
spin_lock(&flush_track_timer_lock);
if (flush_track_timer_stopped[nr]) {
spin_unlock(&flush_track_timer_lock);
return;
}
spin_unlock(&flush_track_timer_lock);
writefromint = 1;
if (!try_fdc(nr)) {
/* we might block in an interrupt, so try again later */
flush_track_timer[nr].expires = jiffies + 1;
add_timer(flush_track_timer + nr);
spin_lock(&flush_track_timer_lock);
if (!flush_track_timer_stopped[nr]) {
flush_track_timer[nr].expires = jiffies + 1;
add_timer(flush_track_timer + nr);
}
spin_unlock(&flush_track_timer_lock);
return;
}
get_fdc(nr);
@@ -1421,6 +1434,35 @@ static int non_int_flush_track (unsigned long nr)
return 1;
}
static void flush_track_timer_stop(unsigned int drive)
{
unsigned long flags;
spin_lock_irqsave(&flush_track_timer_lock, flags);
flush_track_timer_stopped[drive] = true;
spin_unlock_irqrestore(&flush_track_timer_lock, flags);
timer_delete_sync(flush_track_timer + drive);
}
static void flush_track_timer_start(unsigned int drive)
{
unsigned long flags;
spin_lock_irqsave(&flush_track_timer_lock, flags);
flush_track_timer_stopped[drive] = false;
if (unit[drive].dirty)
mod_timer(flush_track_timer + drive, jiffies + 1);
spin_unlock_irqrestore(&flush_track_timer_lock, flags);
}
static void flush_track_timer_sync(unsigned int drive)
{
flush_track_timer_stop(drive);
non_int_flush_track(drive);
flush_track_timer_start(drive);
}
static int get_track(int drive, int track)
{
int error, errcnt;
@@ -1435,8 +1477,7 @@ static int get_track(int drive, int track)
}
if (unit[drive].dirty == 1) {
timer_delete(flush_track_timer + drive);
non_int_flush_track (drive);
flush_track_timer_sync(drive);
}
errcnt = 0;
while (errcnt < MAX_ERRORS) {
@@ -1562,6 +1603,12 @@ static int fd_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
case FDFMTTRK:
if (param < p->type->tracks * p->type->heads)
{
flush_track_timer_stop(drive);
if (unit[drive].dirty == 1 &&
!non_int_flush_track(drive)) {
flush_track_timer_start(drive);
return -EIO;
}
get_fdc(drive);
if (fd_seek(drive,param) != 0){
memset(p->trackbuf, FD_FILL_BYTE,
@@ -1570,6 +1617,7 @@ static int fd_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
}
floppy_off(drive);
rel_fdc();
flush_track_timer_start(drive);
}
else
return -EINVAL;
@@ -1591,8 +1639,7 @@ static int fd_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
case FDDEFPRM:
return -EINVAL;
case FDFLUSH: /* unconditionally, even if not needed */
timer_delete(flush_track_timer + drive);
non_int_flush_track(drive);
flush_track_timer_sync(drive);
break;
#ifdef RAW_IOCTL
case IOCTL_RAW_TRACK:
@@ -1611,9 +1658,15 @@ static int fd_ioctl(struct block_device *bdev, blk_mode_t mode,
unsigned int cmd, unsigned long param)
{
int ret;
unsigned int memflags = 0;
bool freeze_queue = cmd == FDFMTTRK || cmd == FDFLUSH;
mutex_lock(&amiflop_mutex);
if (freeze_queue)
memflags = blk_mq_freeze_queue(bdev->bd_disk->queue);
ret = fd_locked_ioctl(bdev, mode, cmd, param);
if (freeze_queue)
blk_mq_unfreeze_queue(bdev->bd_disk->queue, memflags);
mutex_unlock(&amiflop_mutex);
return ret;
@@ -1711,11 +1764,12 @@ static void floppy_release(struct gendisk *disk)
{
struct amiga_floppy_struct *p = disk->private_data;
int drive = p - unit;
unsigned int memflags;
mutex_lock(&amiflop_mutex);
memflags = blk_mq_freeze_queue(disk->queue);
if (unit[drive].dirty == 1) {
timer_delete(flush_track_timer + drive);
non_int_flush_track (drive);
flush_track_timer_sync(drive);
}
if (!fd_ref[drive]--) {
@@ -1725,6 +1779,7 @@ static void floppy_release(struct gendisk *disk)
#ifdef MODULE
floppy_off (drive);
#endif
blk_mq_unfreeze_queue(disk->queue, memflags);
mutex_unlock(&amiflop_mutex);
}
drivers/block/aoe/aoenet.c
+6 -8
View File
@@ -131,9 +131,7 @@ static int
aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt, struct net_device *orig_dev)
{
struct aoe_hdr *h;
struct aoe_atahdr *ah;
u32 n;
int sn;
if (dev_net(ifp) != &init_net)
goto exit;
@@ -144,12 +142,8 @@ aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt,
if (!is_aoe_netif(ifp))
goto exit;
skb_push(skb, ETH_HLEN); /* (1) */
sn = sizeof(*h) + sizeof(*ah);
if (skb->len >= sn) {
sn -= skb_headlen(skb);
if (sn > 0 && !__pskb_pull_tail(skb, sn))
goto exit;
}
if (!pskb_may_pull(skb, sizeof(*h)))
goto exit;
h = (struct aoe_hdr *) skb->data;
n = get_unaligned_be32(&h->tag);
if ((h->verfl & AOEFL_RSP) == 0 || (n & 1<<31))
@@ -171,10 +165,14 @@ aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt,
switch (h->cmd) {
case AOECMD_ATA:
if (!pskb_may_pull(skb, sizeof(*h) + sizeof(struct aoe_atahdr)))
goto exit;
/* ata_rsp may keep skb for later processing or give it back */
skb = aoecmd_ata_rsp(skb);
break;
case AOECMD_CFG:
if (!pskb_may_pull(skb, sizeof(*h) + sizeof(struct aoe_cfghdr)))
goto exit;
aoecmd_cfg_rsp(skb);
break;
default:
drivers/block/floppy.c
+34 -22
View File
@@ -500,7 +500,7 @@ static int probing;
#define FD_COMMAND_ERROR 2
#define FD_COMMAND_OKAY 3
static volatile int command_status = FD_COMMAND_NONE;
static int command_status = FD_COMMAND_NONE;
static unsigned long fdc_busy;
static DECLARE_WAIT_QUEUE_HEAD(fdc_wait);
static DECLARE_WAIT_QUEUE_HEAD(command_done);
@@ -599,7 +599,8 @@ static inline void fdc_outb(unsigned char value, int fdc, int reg)
static inline bool drive_no_geom(int drive)
{
return !current_type[drive] && !ITYPE(drive_state[drive].fd_device);
return !READ_ONCE(current_type[drive]) &&
!ITYPE(drive_state[drive].fd_device);
}
#ifndef fd_eject
@@ -638,7 +639,7 @@ static const char *timeout_message;
static void is_alive(const char *func, const char *message)
{
/* this routine checks whether the floppy driver is "alive" */
if (test_bit(0, &fdc_busy) && command_status < 2 &&
if (test_bit(0, &fdc_busy) && READ_ONCE(command_status) < 2 &&
!delayed_work_pending(&fd_timeout)) {
DPRINT("%s: timeout handler died. %s\n", func, message);
}
@@ -890,7 +891,7 @@ static int lock_fdc(int drive)
if (wait_event_interruptible(fdc_wait, !test_and_set_bit(0, &fdc_busy)))
return -EINTR;
command_status = FD_COMMAND_NONE;
WRITE_ONCE(command_status, FD_COMMAND_NONE);
reschedule_timeout(drive, "lock fdc");
set_fdc(drive);
@@ -904,7 +905,7 @@ static void unlock_fdc(void)
DPRINT("FDC access conflict!\n");
raw_cmd = NULL;
command_status = FD_COMMAND_NONE;
WRITE_ONCE(command_status, FD_COMMAND_NONE);
cancel_delayed_work(&fd_timeout);
do_floppy = NULL;
cont = NULL;
@@ -988,7 +989,9 @@ static void (*floppy_work_fn)(void);
static void floppy_work_workfn(struct work_struct *work)
{
floppy_work_fn();
void (*fn)(void) = READ_ONCE(floppy_work_fn);
fn();
}
static DECLARE_WORK(floppy_work, floppy_work_workfn);
@@ -997,7 +1000,7 @@ static void schedule_bh(void (*handler)(void))
{
WARN_ON(work_pending(&floppy_work));
floppy_work_fn = handler;
WRITE_ONCE(floppy_work_fn, handler);
queue_work(floppy_wq, &floppy_work);
}
@@ -1784,6 +1787,9 @@ static void reset_interrupt(void)
{
debugt(__func__, "");
result(current_fdc); /* get the status ready for set_fdc */
/* Stale work: unlock_fdc()/do_wakeup() may have cleared cont. */
if (!cont)
return;
if (fdc_state[current_fdc].reset) {
pr_info("reset set in interrupt, calling %ps\n", cont->error);
cont->error(); /* a reset just after a reset. BAD! */
@@ -1862,7 +1868,7 @@ static void show_floppy(int fdc)
pr_info("cont=%p\n", cont);
pr_info("current_req=%p\n", current_req);
pr_info("command_status=%d\n", command_status);
pr_info("command_status=%d\n", READ_ONCE(command_status));
pr_info("\n");
}
@@ -1989,7 +1995,7 @@ static void do_wakeup(void)
{
reschedule_timeout(MAXTIMEOUT, "do wakeup");
cont = NULL;
command_status += 2;
WRITE_ONCE(command_status, READ_ONCE(command_status) + 2);
wake_up(&command_done);
}
@@ -2017,11 +2023,12 @@ static int wait_til_done(void (*handler)(void), bool interruptible)
schedule_bh(handler);
if (interruptible)
wait_event_interruptible(command_done, command_status >= 2);
wait_event_interruptible(command_done,
READ_ONCE(command_status) >= 2);
else
wait_event(command_done, command_status >= 2);
wait_event(command_done, READ_ONCE(command_status) >= 2);
if (command_status < 2) {
if (READ_ONCE(command_status) < 2) {
cancel_activity();
cont = &intr_cont;
reset_fdc();
@@ -2029,18 +2036,18 @@ static int wait_til_done(void (*handler)(void), bool interruptible)
}
if (fdc_state[current_fdc].reset)
command_status = FD_COMMAND_ERROR;
if (command_status == FD_COMMAND_OKAY)
WRITE_ONCE(command_status, FD_COMMAND_ERROR);
if (READ_ONCE(command_status) == FD_COMMAND_OKAY)
ret = 0;
else
ret = -EIO;
command_status = FD_COMMAND_NONE;
WRITE_ONCE(command_status, FD_COMMAND_NONE);
return ret;
}
static void generic_done(int result)
{
command_status = result;
WRITE_ONCE(command_status, result);
cont = &wakeup_cont;
}
@@ -2276,11 +2283,10 @@ static void request_done(int uptodate)
{
struct request *req = current_req;
int block;
char msg[sizeof("request done ") + sizeof(int) * 3];
probing = 0;
snprintf(msg, sizeof(msg), "request done %d", uptodate);
reschedule_timeout(MAXTIMEOUT, msg);
reschedule_timeout(MAXTIMEOUT,
uptodate ? "request done 1" : "request done 0");
if (!req) {
pr_info("floppy.c: no request in request_done\n");
@@ -2871,7 +2877,7 @@ static blk_status_t floppy_queue_rq(struct blk_mq_hw_ctx *hctx,
list_add_tail(&bd->rq->queuelist, &floppy_reqs);
spin_unlock_irq(&floppy_lock);
command_status = FD_COMMAND_NONE;
WRITE_ONCE(command_status, FD_COMMAND_NONE);
__reschedule_timeout(MAXTIMEOUT, "fd_request");
set_fdc(0);
process_fd_request();
@@ -3601,7 +3607,7 @@ struct compat_floppy_drive_params {
};
struct compat_floppy_drive_struct {
signed char flags;
compat_ulong_t flags;
compat_ulong_t spinup_date;
compat_ulong_t select_date;
compat_ulong_t first_read_date;
@@ -3785,6 +3791,7 @@ static int compat_getdrvstat(int drive, bool poll,
goto Eintr;
process_fd_request();
}
v.flags = drive_state[drive].flags;
v.spinup_date = drive_state[drive].spinup_date;
v.select_date = drive_state[drive].select_date;
v.first_read_date = drive_state[drive].first_read_date;
@@ -4729,8 +4736,11 @@ static int __init do_floppy_init(void)
err = device_add_disk(&floppy_device[drive].dev,
disks[drive][0], NULL);
if (err)
if (err) {
platform_device_unregister(&floppy_device[drive]);
registered[drive] = false;
goto out_remove_drives;
}
}
return 0;
@@ -4744,6 +4754,8 @@ out_remove_drives:
}
}
out_release_dma:
/* Drain the queued redo_fd_request() before releasing IRQ and DMA. */
flush_workqueue(floppy_wq);
if (atomic_read(&usage_count))
floppy_release_irq_and_dma();
out_unreg_driver:
drivers/block/loop.c
+4 -108
View File
@@ -105,7 +105,7 @@ static DEFINE_MUTEX(loop_validate_mutex);
*
* Since loop_validate_file() traverses on other "struct loop_device" if
* is_loop_device() is true, we need a global lock for serializing concurrent
* loop_configure()/loop_change_fd()/__loop_clr_fd() calls.
* loop_configure()/__loop_clr_fd() calls.
*/
static int loop_global_lock_killable(struct loop_device *lo, bool global)
{
@@ -537,109 +537,6 @@ static int loop_check_backing_file(struct file *file)
return 0;
}
/*
* loop_change_fd switched the backing store of a loopback device to
* a new file. This is useful for operating system installers to free up
* the original file and in High Availability environments to switch to
* an alternative location for the content in case of server meltdown.
* This can only work if the loop device is used read-only, and if the
* new backing store is the same size and type as the old backing store.
*/
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
unsigned int arg)
{
struct file *file = fget(arg);
struct file *old_file;
unsigned int memflags;
int error;
bool partscan;
bool is_loop;
if (!file)
return -EBADF;
error = loop_check_backing_file(file);
if (error) {
fput(file);
return error;
}
/* suppress uevents while reconfiguring the device */
dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
is_loop = is_loop_device(file);
error = loop_global_lock_killable(lo, is_loop);
if (error)
goto out_putf;
error = -ENXIO;
if (lo->lo_state != Lo_bound)
goto out_err;
/* the loop device has to be read-only */
error = -EINVAL;
if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
goto out_err;
error = loop_validate_file(file, bdev);
if (error)
goto out_err;
old_file = lo->lo_backing_file;
error = -EINVAL;
/* size of the new backing store needs to be the same */
if (lo_calculate_size(lo, file) != lo_calculate_size(lo, old_file))
goto out_err;
/*
* We might switch to direct I/O mode for the loop device, write back
* all dirty data the page cache now that so that the individual I/O
* operations don't have to do that.
*/
vfs_fsync(file, 0);
/* and ... switch */
disk_force_media_change(lo->lo_disk);
memflags = blk_mq_freeze_queue(lo->lo_queue);
mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
loop_assign_backing_file(lo, file);
loop_update_dio(lo);
blk_mq_unfreeze_queue(lo->lo_queue, memflags);
partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
loop_global_unlock(lo, is_loop);
/*
* Flush loop_validate_file() before fput(), for l->lo_backing_file
* might be pointing at old_file which might be the last reference.
*/
if (!is_loop) {
mutex_lock(&loop_validate_mutex);
mutex_unlock(&loop_validate_mutex);
}
/*
* We must drop file reference outside of lo_mutex as dropping
* the file ref can take open_mutex which creates circular locking
* dependency.
*/
fput(old_file);
dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
if (partscan)
loop_reread_partitions(lo);
error = 0;
done:
kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
return error;
out_err:
loop_global_unlock(lo, is_loop);
out_putf:
fput(file);
dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
goto done;
}
/* loop sysfs attributes */
static ssize_t loop_attr_show(struct device *dev, char *page,
@@ -1215,12 +1112,11 @@ static int loop_clr_fd(struct loop_device *lo)
/*
* Since lo_ioctl() is called without locks held, it is possible that
* loop_configure()/loop_change_fd() and loop_clr_fd() run in parallel.
* loop_configure() and loop_clr_fd() run in parallel.
*
* Therefore, use global lock when setting Lo_rundown state in order to
* make sure that loop_validate_file() will fail if the "struct file"
* which loop_configure()/loop_change_fd() found via fget() was this
* loop device.
* which loop_configure() found via fget() was this loop device.
*/
err = loop_global_lock_killable(lo, true);
if (err)
@@ -1566,7 +1462,7 @@ static int lo_ioctl(struct block_device *bdev, blk_mode_t mode,
return loop_configure(lo, mode, bdev, &config);
}
case LOOP_CHANGE_FD:
return loop_change_fd(lo, bdev, arg);
return -EOPNOTSUPP;
case LOOP_CLR_FD:
return loop_clr_fd(lo);
case LOOP_SET_STATUS:
drivers/block/rnull/configfs.rs
+30 -14
View File
@@ -2,15 +2,27 @@
use super::NullBlkDevice;
use kernel::{
block::mq::gen_disk::{GenDisk, GenDiskBuilder},
configfs::{self, AttributeOperations},
block::mq::gen_disk::{
GenDisk,
GenDiskBuilder, //
},
configfs::{
self,
AttributeOperations, //
},
configfs_attrs,
fmt::{self, Write as _},
fmt::{
self,
Write as _, //
},
new_mutex,
page::PAGE_SIZE,
prelude::*,
str::{kstrtobool_bytes, CString},
sync::Mutex,
str::{
kstrtobool_bytes,
CString, //
},
sync::Mutex, //
};
pub(crate) fn subsystem() -> impl PinInit<kernel::configfs::Subsystem<Config>, Error> {
@@ -35,7 +47,7 @@ impl AttributeOperations<0> for Config {
fn show(_this: &Config, page: &mut [u8; PAGE_SIZE]) -> Result<usize> {
let mut writer = kernel::str::Formatter::new(page);
writer.write_str("blocksize,size,rotational,irqmode\n")?;
writer.write_str("blocksize,size,rotational,irqmode,power\n")?;
Ok(writer.bytes_written())
}
}
@@ -174,7 +186,8 @@ impl configfs::AttributeOperations<1> for DeviceConfig {
}
fn store(this: &DeviceConfig, page: &[u8]) -> Result {
if this.data.lock().powered {
let mut guard = this.data.lock();
if guard.powered {
return Err(EBUSY);
}
@@ -182,7 +195,7 @@ impl configfs::AttributeOperations<1> for DeviceConfig {
let value = text.parse::<u32>().map_err(|_| EINVAL)?;
GenDiskBuilder::validate_block_size(value)?;
this.data.lock().block_size = value;
guard.block_size = value;
Ok(())
}
}
@@ -204,11 +217,12 @@ impl configfs::AttributeOperations<2> for DeviceConfig {
}
fn store(this: &DeviceConfig, page: &[u8]) -> Result {
if this.data.lock().powered {
let mut guard = this.data.lock();
if guard.powered {
return Err(EBUSY);
}
this.data.lock().rotational = kstrtobool_bytes(page)?;
guard.rotational = kstrtobool_bytes(page)?;
Ok(())
}
@@ -225,14 +239,15 @@ impl configfs::AttributeOperations<3> for DeviceConfig {
}
fn store(this: &DeviceConfig, page: &[u8]) -> Result {
if this.data.lock().powered {
let mut guard = this.data.lock();
if guard.powered {
return Err(EBUSY);
}
let text = core::str::from_utf8(page)?.trim();
let value = text.parse::<u64>().map_err(|_| EINVAL)?;
this.data.lock().capacity_mib = value;
guard.capacity_mib = value;
Ok(())
}
}
@@ -248,14 +263,15 @@ impl configfs::AttributeOperations<4> for DeviceConfig {
}
fn store(this: &DeviceConfig, page: &[u8]) -> Result {
if this.data.lock().powered {
let mut guard = this.data.lock();
if guard.powered {
return Err(EBUSY);
}
let text = core::str::from_utf8(page)?.trim();
let value = text.parse::<u8>().map_err(|_| EINVAL)?;
this.data.lock().irq_mode = IRQMode::try_from(value)?;
guard.irq_mode = IRQMode::try_from(value)?;
Ok(())
}
}
drivers/block/rnull/rnull.rs
+11 -4
View File
@@ -10,12 +10,19 @@ use kernel::{
self,
mq::{
self,
gen_disk::{self, GenDisk},
Operations, TagSet,
},
gen_disk::{
self,
GenDisk, //
},
Operations,
TagSet, //
}, //
},
prelude::*,
sync::{aref::ARef, Arc},
sync::{
aref::ARef,
Arc, //
}, //
};
module! {
drivers/block/swim.c
+240 -217
View File
@@ -2,12 +2,20 @@
/*
* Driver for SWIM (Sander Woz Integrated Machine) floppy controller
*
* Copyright (C) 2004,2008 Laurent Vivier <Laurent@lvivier.info>
* Copyright (C) 2004, 2008 Laurent Vivier <Laurent@lvivier.info>
*
* based on Alastair Bridgewater SWIM analysis, 2001
* based on SWIM3 driver (c) Paul Mackerras, 1996
* based on netBSD IWM driver (c) 1997, 1998 Hauke Fath.
*
* See also:
* Inside Macintosh, vol. III, ch. 2
* https://archive.org/details/SWIMDesignDocs
* NEC uPD72070 FDC datasheet
* mkLinux source file swimiiicommonhal.c
* http://www.mac.linux-m68k.org/devel/iwm.php.html
* MAME source file sonydriv.cpp
*
* 2004-08-21 (lv) - Initial implementation
* 2008-10-30 (lv) - Port to 2.6
*/
@@ -19,6 +27,7 @@
#include <linux/major.h>
#include <linux/mutex.h>
#include <linux/hdreg.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/platform_device.h>
@@ -60,7 +69,7 @@ struct swim {
REG(read_handshake)
} __attribute__((packed));
#define swim_write(base, reg, v) out_8(&(base)->write_##reg, (v))
#define swim_write(base, reg, v) out_8(&(base)->write_##reg, (v))
#define swim_read(base, reg) in_8(&(base)->read_##reg)
/* IWM registers */
@@ -84,62 +93,54 @@ struct iwm {
REG(q7H)
} __attribute__((packed));
#define iwm_write(base, reg, v) out_8(&(base)->reg, (v))
#define iwm_write(base, reg, v) out_8(&(base)->reg, (v))
#define iwm_read(base, reg) in_8(&(base)->reg)
/* bits in phase register */
/* Bits in phase register */
#define SEEK_POSITIVE 0x070
#define SEEK_NEGATIVE 0x074
#define STEP 0x071
#define MOTOR_ON 0x072
#define MOTOR_OFF 0x076
#define INDEX 0x073
#define EJECT 0x077
#define SETMFM 0x171
#define SETGCR 0x175
#define RELAX 0x033
#define LSTRB 0x008
#define CA_MASK 0x077
#define RELAX 0x03
#define LSTRB 0x08
#define CA_MASK 0x07
#define PHASE_PIN_DIR 0xF0
/* Select values for swim_select and swim_readbit */
#define READ_DATA_0 0x074
#define ONEMEG_DRIVE 0x075
#define SINGLE_SIDED 0x076
#define DRIVE_PRESENT 0x077
#define DISK_IN 0x170
#define WRITE_PROT 0x171
#define TRACK_ZERO 0x172
#define TACHO 0x173
#define READ_DATA_1 0x174
#define GCR_MODE 0x175
#define SEEK_COMPLETE 0x176
#define TWOMEG_MEDIA 0x177
#define SEL_MASK 0x100
#define SEEK_POSITIVE 0x000
#define SEEK_NEGATIVE 0x004
#define STEP 0x001
#define MOTOR_ON 0x002
#define MOTOR_OFF 0x006
#define INDEX 0x003
#define EJECT 0x007
#define SETMFM 0x101
#define SETGCR 0x105
#define STEPPING 0x001
#define READ_DATA_0 0x004
#define ONEMEG_DRIVE 0x005
#define SINGLE_SIDED 0x006
#define DRIVE_PRESENT 0x007
#define DISK_IN 0x100
#define WRITE_PROT 0x101
#define TRACK_ZERO 0x102
#define TACHO 0x103
#define READ_DATA_1 0x104
#define GCR_MODE 0x105
#define READY 0x106
#define TWOMEG_MEDIA 0x107
/* Bits in handshake register */
#define MARK_BYTE 0x01
#define CRC_ZERO 0x02
#define RDDATA 0x04
#define SENSE 0x08
#define MOTEN 0x10
#define ERROR 0x20
#define DAT2BYTE 0x40
#define DAT1BYTE 0x80
/* bits in setup register */
#define S_INV_WDATA 0x01
#define S_3_5_SELECT 0x02
#define S_GCR 0x04
#define S_FCLK_DIV2 0x08
#define S_ERROR_CORR 0x10
#define S_IBM_DRIVE 0x20
#define S_GCR_WRITE 0x40
#define S_TIMEOUT 0x80
/* bits in mode register */
@@ -148,14 +149,13 @@ struct iwm {
#define ENBL2 0x04
#define ACTION 0x08
#define WRITE_MODE 0x10
#define HEDSEL 0x20
#define ISM_SELECT 0x40
#define MOTON 0x80
/*----------------------------------------------------------------------------*/
enum drive_location {
INTERNAL_DRIVE = 0x02,
EXTERNAL_DRIVE = 0x04,
NO_DRIVE = 0,
INTERNAL_DRIVE = BIT(1),
EXTERNAL_DRIVE = BIT(2),
};
enum media_type {
@@ -168,23 +168,23 @@ struct floppy_state {
/* physical properties */
enum drive_location location; /* internal or external drive */
int head_number; /* single- or double-sided drive */
int head_number; /* single- or double-sided drive */
/* media */
int disk_in;
int ejected;
enum media_type type;
int write_protected;
int disk_in;
int ejected;
enum media_type type;
int write_protected;
int total_secs;
int secpercyl;
int secpertrack;
unsigned int total_secs;
unsigned int secpercyl;
unsigned int secpertrack;
/* in-use information */
int track;
int ref_count;
int track;
int ref_count;
bool registered;
struct gendisk *disk;
@@ -209,7 +209,6 @@ enum head {
struct swim_priv {
struct swim __iomem *base;
spinlock_t lock;
int floppy_count;
struct floppy_state unit[FD_MAX_UNIT];
};
@@ -220,20 +219,24 @@ extern int swim_read_sector_data(struct swim __iomem *base,
unsigned char *data);
static DEFINE_MUTEX(swim_mutex);
static inline void set_swim_mode(struct swim __iomem *base, int enable)
static void set_swim_mode(struct swim __iomem *base, int enable)
{
struct iwm __iomem *iwm_base;
struct iwm __iomem *iwm_base = (struct iwm __iomem *)base;
unsigned long flags;
if (!enable) {
swim_write(base, mode0, 0xf8);
swim_write(base, mode0, ACTION);
swim_write(base, mode0, ENBL1 | ENBL2 | MOTON);
swim_write(base, mode0, ISM_SELECT);
iwm_read(iwm_base, q7L);
return;
}
iwm_base = (struct iwm __iomem *)base;
local_irq_save(flags);
iwm_read(iwm_base, q7L);
iwm_read(iwm_base, q6L);
iwm_read(iwm_base, mtrOff);
iwm_read(iwm_base, q6H);
@@ -245,7 +248,7 @@ static inline void set_swim_mode(struct swim __iomem *base, int enable)
local_irq_restore(flags);
}
static inline int get_swim_mode(struct swim __iomem *base)
static int get_swim_mode(struct swim __iomem *base)
{
unsigned long flags;
@@ -269,11 +272,11 @@ is_iwm:
static inline void swim_select(struct swim __iomem *base, int sel)
{
swim_write(base, phase, RELAX);
swim_write(base, phase, RELAX | PHASE_PIN_DIR);
via1_set_head(sel & 0x100);
via1_set_head(sel & SEL_MASK);
swim_write(base, phase, sel & CA_MASK);
swim_write(base, phase, (sel & CA_MASK) | PHASE_PIN_DIR);
}
static inline void swim_action(struct swim __iomem *base, int action)
@@ -284,15 +287,15 @@ static inline void swim_action(struct swim __iomem *base, int action)
swim_select(base, action);
udelay(1);
swim_write(base, phase, (LSTRB<<4) | LSTRB);
udelay(1);
swim_write(base, phase, (LSTRB<<4) | ((~LSTRB) & 0x0F));
swim_write(base, phase, LSTRB | action | PHASE_PIN_DIR);
udelay(2);
swim_write(base, phase, action | PHASE_PIN_DIR);
udelay(1);
local_irq_restore(flags);
}
static inline int swim_readbit(struct swim __iomem *base, int bit)
static inline bool swim_readbit(struct swim __iomem *base, int bit)
{
int stat;
@@ -305,125 +308,116 @@ static inline int swim_readbit(struct swim __iomem *base, int bit)
return (stat & SENSE) == 0;
}
static inline void swim_drive(struct swim __iomem *base,
#define swim_readbit_timeout(base, bit, val, timeout_us) \
poll_timeout_us(, swim_readbit(base, bit) == val, 1000, timeout_us, false)
#define swim_READY_timeout(base) \
({ \
int ret = swim_readbit_timeout(base, READY, true, 1000 * 1000); \
if (ret) \
printk(KERN_DEBUG "%s: drive not ready\n", __func__); \
ret; \
})
static void swim_drive(struct swim __iomem *base,
enum drive_location location)
{
if (location == INTERNAL_DRIVE) {
swim_write(base, mode0, EXTERNAL_DRIVE); /* clear drive 1 bit */
swim_write(base, mode1, INTERNAL_DRIVE); /* set drive 0 bit */
swim_write(base, mode1, MOTON);
} else if (location == EXTERNAL_DRIVE) {
swim_write(base, mode0, INTERNAL_DRIVE); /* clear drive 0 bit */
swim_write(base, mode1, EXTERNAL_DRIVE); /* set drive 1 bit */
swim_write(base, mode1, MOTON);
} else {
swim_write(base, mode0, INTERNAL_DRIVE);
swim_write(base, mode0, EXTERNAL_DRIVE);
swim_write(base, mode0, MOTON);
}
udelay(1);
}
static inline void swim_motor(struct swim __iomem *base,
enum motor_action action)
static void swim_motor(struct swim __iomem *base,
enum motor_action action)
{
if (action == ON) {
int i;
swim_action(base, MOTOR_ON);
for (i = 0; i < 2*HZ; i++) {
swim_select(base, RELAX);
if (swim_readbit(base, MOTOR_ON))
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(1);
}
swim_readbit_timeout(base, MOTOR_ON, true, 2000 * 1000);
} else if (action == OFF) {
swim_action(base, MOTOR_OFF);
swim_select(base, RELAX);
swim_write(base, phase, RELAX | PHASE_PIN_DIR);
}
}
static inline void swim_eject(struct swim __iomem *base)
static void swim_eject(struct swim __iomem *base)
{
int i;
swim_action(base, EJECT);
for (i = 0; i < 2*HZ; i++) {
swim_select(base, RELAX);
if (!swim_readbit(base, DISK_IN))
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(1);
}
swim_select(base, RELAX);
swim_readbit_timeout(base, DISK_IN, false, 2000 * 1000);
msleep(1);
}
static inline void swim_head(struct swim __iomem *base, enum head head)
{
/* wait drive is ready */
if (head == UPPER_HEAD)
swim_select(base, READ_DATA_1);
else if (head == LOWER_HEAD)
swim_select(base, READ_DATA_0);
}
static inline int swim_step(struct swim __iomem *base)
static int swim_step(struct swim __iomem *base)
{
int wait;
swim_action(base, STEP);
for (wait = 0; wait < HZ; wait++) {
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(1);
swim_select(base, RELAX);
if (!swim_readbit(base, STEP))
return 0;
}
return -1;
udelay(150);
return swim_readbit_timeout(base, STEPPING, false, 20 * 1000);
}
static inline int swim_track00(struct swim __iomem *base)
static int swim_track00(struct swim __iomem *base)
{
int try;
swim_action(base, SEEK_NEGATIVE);
for (try = 0; try < 100; try++) {
msleep(3);
swim_select(base, RELAX);
if (swim_readbit(base, TRACK_ZERO))
break;
return 0;
if (swim_step(base))
return -1;
break;
}
if (swim_readbit(base, TRACK_ZERO))
return 0;
pr_err("swim: track zero recalibration failed\n");
return -1;
}
static inline int swim_seek(struct swim __iomem *base, int step)
static int swim_seek(struct swim __iomem *base, int step)
{
if (step == 0)
return 0;
if (step < 0) {
swim_action(base, SEEK_NEGATIVE);
step = -step;
} else
} else if (step > 0)
swim_action(base, SEEK_POSITIVE);
udelay(1);
for ( ; step > 0; step--) {
swim_READY_timeout(base);
if (step == 0)
return 0;
for (; step > 0; step--) {
if (swim_step(base))
return -1;
}
msleep(30);
swim_READY_timeout(base);
return 0;
}
static inline int swim_track(struct floppy_state *fs, int track)
static int swim_track(struct floppy_state *fs, int track)
{
struct swim __iomem *base = fs->swd->base;
int ret;
@@ -445,8 +439,10 @@ static int floppy_eject(struct floppy_state *fs)
struct swim __iomem *base = fs->swd->base;
swim_drive(base, fs->location);
swim_track(fs, 40);
swim_motor(base, OFF);
swim_eject(base);
swim_drive(base, NO_DRIVE);
fs->disk_in = 0;
fs->ejected = 1;
@@ -454,70 +450,79 @@ static int floppy_eject(struct floppy_state *fs)
return 0;
}
static inline int swim_read_sector(struct floppy_state *fs,
int side, int track,
int sector, unsigned char *buffer)
static unsigned int swim_read_sector_range(struct floppy_state *fs,
unsigned int side, unsigned int track,
unsigned int start, unsigned int count,
unsigned char *buffer)
{
struct swim __iomem *base = fs->swd->base;
unsigned long flags;
struct sector_header header;
int ret = -1;
short i;
unsigned int i, bits = 0;
swim_track(fs, track);
swim_write(base, mode1, MOTON);
swim_head(base, side);
swim_write(base, mode0, side);
if (count > 0) {
count = min(count, fs->secpertrack);
bits = GENMASK(count - 1, 0);
}
local_irq_save(flags);
for (i = 0; i < 36; i++) {
ret = swim_read_sector_header(base, &header);
if (!ret && (header.sector == sector)) {
/* found */
ret = swim_read_sector_data(base, buffer);
for (i = 0; i < fs->secpertrack + 1; i++) {
if (bits == 0) /* All sectors were read ok */
break;
if (swim_read_sector_header(base, &header) == 0 &&
swim_read(base, error) == 0) {
unsigned int offset = header.sector - start;
unsigned char *buf = NULL;
int len;
if (header.track == track && header.side == side &&
header.size == 2 && header.sector >= start &&
header.sector < start + count &&
(bits & BIT(offset)))
buf = buffer + 512 * offset;
len = swim_read_sector_data(base, buf);
if (swim_read(base, error) == 0 && buf && len == 512)
bits &= ~BIT(offset);
}
}
local_irq_restore(flags);
swim_write(base, mode0, MOTON);
if ((header.side != side) || (header.track != track) ||
(header.sector != sector))
return 0;
return ret;
return bits ? ffs(bits) - 1 : count; /* No. of contiguous ok sectors */
}
static blk_status_t floppy_read_sectors(struct floppy_state *fs,
int req_sector, int sectors_nb,
unsigned char *buffer)
unsigned int req_sector,
unsigned int sectors_nb,
unsigned char *buffer)
{
struct swim __iomem *base = fs->swd->base;
int ret;
int side, track, sector;
int i, try;
unsigned int try = 0;
swim_drive(base, fs->location);
for (i = req_sector; i < req_sector + sectors_nb; i++) {
int x;
track = i / fs->secpercyl;
x = i % fs->secpercyl;
side = x / fs->secpertrack;
sector = x % fs->secpertrack + 1;
swim_READY_timeout(base);
try = 5;
do {
ret = swim_read_sector(fs, side, track, sector,
buffer);
if (try-- == 0)
return BLK_STS_IOERR;
} while (ret != 512);
while (sectors_nb) {
unsigned int cyl, x, head, sector, n, n_ok;
buffer += ret;
cyl = req_sector / fs->secpercyl;
x = req_sector % fs->secpercyl;
head = (x >= fs->secpertrack) ? 1 : 0;
sector = x % fs->secpertrack;
n = min(sectors_nb, fs->secpertrack - sector);
swim_track(fs, cyl);
swim_head(base, head);
n_ok = swim_read_sector_range(fs, head, cyl, sector + 1, n, buffer);
if (n_ok == n)
try = 0;
else if (++try >= 5)
return BLK_STS_IOERR;
buffer += 512 * n_ok;
sectors_nb -= n_ok;
req_sector += n_ok;
}
return 0;
@@ -527,12 +532,10 @@ static blk_status_t swim_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
struct floppy_state *fs = hctx->queue->queuedata;
struct swim_priv *swd = fs->swd;
struct request *req = bd->rq;
blk_status_t err;
if (!spin_trylock_irq(&swd->lock))
return BLK_STS_DEV_RESOURCE;
mutex_lock(&swim_mutex);
blk_mq_start_request(req);
@@ -550,16 +553,16 @@ static blk_status_t swim_queue_rq(struct blk_mq_hw_ctx *hctx,
err = BLK_STS_OK;
out:
spin_unlock_irq(&swd->lock);
mutex_unlock(&swim_mutex);
return err;
}
static struct floppy_struct floppy_type[4] = {
{ 0, 0, 0, 0, 0, 0x00, 0x00, 0x00, 0x00, NULL }, /* no testing */
{ 0, 0, 0, 0, 0, 0x00, 0x00, 0x00, 0x00, NULL }, /* no testing */
{ 720, 9, 1, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 360KB SS 3.5"*/
{ 1440, 9, 2, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 720KB 3.5" */
{ 2880, 18, 2, 80, 0, 0x1B, 0x00, 0xCF, 0x6C, NULL }, /* 1.44MB 3.5" */
{ 1440, 9, 2, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 720KB 3.5" */
{ 2880, 18, 2, 80, 0, 0x1B, 0x00, 0xCF, 0x6C, NULL }, /* 1.44MB 3.5" */
};
static int get_floppy_geometry(struct floppy_state *fs, int type,
@@ -586,23 +589,16 @@ static void setup_medium(struct floppy_state *fs)
if (swim_readbit(base, DISK_IN)) {
struct floppy_struct *g;
fs->disk_in = 1;
fs->write_protected = swim_readbit(base, WRITE_PROT);
if (swim_track00(base))
printk(KERN_ERR
"SWIM: cannot move floppy head to track 0\n");
swim_track00(base);
fs->type = swim_readbit(base, TWOMEG_MEDIA) ?
HD_MEDIA : DD_MEDIA;
HD_MEDIA : DD_MEDIA;
fs->head_number = swim_readbit(base, SINGLE_SIDED) ? 1 : 2;
get_floppy_geometry(fs, 0, &g);
fs->total_secs = g->size;
fs->secpercyl = g->head * g->sect;
fs->secpertrack = g->sect;
fs->track = 0;
} else {
fs->disk_in = 0;
}
@@ -620,11 +616,9 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
fs->ref_count = -1;
else
fs->ref_count++;
swim_write(base, setup, S_IBM_DRIVE | S_FCLK_DIV2);
udelay(10);
swim_drive(base, fs->location);
swim_motor(base, ON);
swim_action(base, SETMFM);
if (fs->ejected)
setup_medium(fs);
if (!fs->disk_in) {
@@ -632,6 +626,11 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
goto out;
}
swim_motor(base, ON);
swim_action(base, SETMFM);
msleep(30);
swim_READY_timeout(base);
set_capacity(fs->disk, fs->total_secs);
if (mode & BLK_OPEN_NDELAY)
@@ -640,7 +639,8 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
if (mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) {
if (disk_check_media_change(disk) && fs->disk_in)
fs->ejected = 0;
if ((mode & BLK_OPEN_WRITE) && fs->write_protected) {
if (mode & BLK_OPEN_WRITE) {
/* Write support isn't implemented. */
err = -EROFS;
goto out;
}
@@ -652,8 +652,10 @@ out:
else if (fs->ref_count > 0)
--fs->ref_count;
if (fs->ref_count == 0)
if (fs->ref_count == 0) {
swim_motor(base, OFF);
swim_drive(base, NO_DRIVE);
}
return err;
}
@@ -679,8 +681,11 @@ static void floppy_release(struct gendisk *disk)
else if (fs->ref_count > 0)
--fs->ref_count;
if (fs->ref_count == 0)
if (fs->ref_count == 0) {
swim_drive(base, fs->location);
swim_motor(base, OFF);
swim_drive(base, NO_DRIVE);
}
mutex_unlock(&swim_mutex);
}
@@ -691,14 +696,15 @@ static int floppy_ioctl(struct block_device *bdev, blk_mode_t mode,
int err;
if ((cmd & 0x80) && !capable(CAP_SYS_ADMIN))
return -EPERM;
return -EPERM;
switch (cmd) {
case FDEJECT:
if (fs->ref_count != 1)
return -EBUSY;
mutex_lock(&swim_mutex);
err = floppy_eject(fs);
if (fs->ref_count == -1 || fs->ref_count == 1)
err = floppy_eject(fs);
else
err = -EBUSY;
mutex_unlock(&swim_mutex);
return err;
@@ -745,26 +751,34 @@ static const struct block_device_operations floppy_fops = {
.check_events = floppy_check_events,
};
static int swim_add_floppy(struct swim_priv *swd, enum drive_location location)
static void swim_add_floppy(struct swim_priv *swd, enum drive_location location)
{
struct floppy_state *fs = &swd->unit[swd->floppy_count];
struct swim __iomem *base = swd->base;
fs->location = location;
swim_drive(base, location);
if (!swim_readbit(base, DRIVE_PRESENT) ||
swim_readbit(base, ONEMEG_DRIVE))
goto out;
if (swim_readbit(base, DISK_IN))
swim_motor(base, ON);
if (swim_track00(base))
goto out;
swim_motor(base, OFF);
fs->swd = swd;
fs->location = location;
fs->type = HD_MEDIA;
fs->head_number = 2;
fs->ref_count = 0;
fs->ejected = 1;
fs->track = 0;
swd->floppy_count++;
return 0;
out:
swim_motor(base, OFF);
}
static const struct blk_mq_ops swim_mq_ops = {
@@ -785,25 +799,38 @@ static void swim_cleanup_floppy_disk(struct floppy_state *fs)
blk_mq_free_tag_set(&fs->tag_set);
}
static int swim_floppy_init(struct swim_priv *swd)
static void swim_set_parameters(struct swim __iomem *base)
{
unsigned int i;
static const u8 mem[] = { 0x18, 0x41, 0x2e, 0x2e, 0x18, 0x18, 0x1b, 0x1b,
0x2f, 0x2f, 0x19, 0x19, 0x97, 0x1b, 0x57, 0x3b, };
swim_write(base, mode0, 0); /* reset parameter memory index */
for (i = 0; i < 16; ++i)
swim_write(base, parameter, mem[i]);
}
static int swim_floppy_init(struct platform_device *pdev)
{
struct swim_priv *swd = platform_get_drvdata(pdev);
bool *data = pdev->dev.platform_data;
bool fast_fclk = data && *data;
struct queue_limits lim = {
.features = BLK_FEAT_ROTATIONAL,
.features = BLK_FEAT_ROTATIONAL,
};
int err;
int drive;
struct swim __iomem *base = swd->base;
swim_write(base, setup, S_IBM_DRIVE | (fast_fclk ? S_FCLK_DIV2 : 0));
swim_set_parameters(base);
/* scan floppy drives */
swim_drive(base, INTERNAL_DRIVE);
if (swim_readbit(base, DRIVE_PRESENT) &&
!swim_readbit(base, ONEMEG_DRIVE))
swim_add_floppy(swd, INTERNAL_DRIVE);
swim_drive(base, EXTERNAL_DRIVE);
if (swim_readbit(base, DRIVE_PRESENT) &&
!swim_readbit(base, ONEMEG_DRIVE))
swim_add_floppy(swd, EXTERNAL_DRIVE);
swim_add_floppy(swd, INTERNAL_DRIVE);
swim_add_floppy(swd, EXTERNAL_DRIVE);
swim_drive(base, NO_DRIVE);
/* register floppy drives */
@@ -814,11 +841,9 @@ static int swim_floppy_init(struct swim_priv *swd)
return -EBUSY;
}
spin_lock_init(&swd->lock);
for (drive = 0; drive < swd->floppy_count; drive++) {
err = blk_mq_alloc_sq_tag_set(&swd->unit[drive].tag_set,
&swim_mq_ops, 2, 0);
&swim_mq_ops, 2, BLK_MQ_F_BLOCKING);
if (err)
goto exit_put_disks;
@@ -830,8 +855,6 @@ static int swim_floppy_init(struct swim_priv *swd)
err = PTR_ERR(swd->unit[drive].disk);
goto exit_put_disks;
}
swd->unit[drive].swd = swd;
}
for (drive = 0; drive < swd->floppy_count; drive++) {
@@ -905,7 +928,7 @@ static int swim_probe(struct platform_device *dev)
swd->base = swim_base;
ret = swim_floppy_init(swd);
ret = swim_floppy_init(dev);
if (ret)
goto out_kfree;
@@ -945,7 +968,7 @@ static void swim_remove(struct platform_device *dev)
static struct platform_driver swim_driver = {
.probe = swim_probe,
.remove = swim_remove,
.driver = {
.driver = {
.name = CARDNAME,
},
};
drivers/block/swim3.c
+3 -5
View File
@@ -887,7 +887,6 @@ static int floppy_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
unsigned int cmd, unsigned long param)
{
struct floppy_state *fs = bdev->bd_disk->private_data;
int err;
if ((cmd & 0x80) && !capable(CAP_SYS_ADMIN))
return -EPERM;
@@ -898,10 +897,9 @@ static int floppy_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
switch (cmd) {
case FDEJECT:
if (fs->ref_count != 1)
return -EBUSY;
err = fd_eject(fs);
return err;
if (fs->ref_count == -1 || fs->ref_count == 1)
return fd_eject(fs);
return -EBUSY;
case FDGETPRM:
if (copy_to_user((void __user *) param, &floppy_type,
sizeof(struct floppy_struct)))
drivers/block/swim_asm.S
+178 -162
View File
@@ -5,7 +5,7 @@
* needs assembly language because is very timing dependent
* this controller exists only on macintosh 680x0 based
*
* Copyright (C) 2004,2008 Laurent Vivier <Laurent@lvivier.info>
* Copyright (C) 2004, 2008 Laurent Vivier <Laurent@lvivier.info>
*
* based on Alastair Bridgewater SWIM analysis, 2001
* based on netBSD IWM driver (c) 1997, 1998 Hauke Fath.
@@ -14,135 +14,160 @@
* 2008-11-05 (lv) - add get_swim_mode
*/
.equ write_data, 0x0000
.equ write_mark, 0x0200
.equ write_CRC, 0x0400
.equ write_parameter,0x0600
.equ write_phase, 0x0800
.equ write_setup, 0x0a00
.equ write_mode0, 0x0c00
.equ write_mode1, 0x0e00
.equ read_data, 0x1000
.equ read_mark, 0x1200
.equ read_error, 0x1400
.equ read_parameter, 0x1600
.equ read_phase, 0x1800
.equ read_setup, 0x1a00
.equ read_status, 0x1c00
.equ read_handshake, 0x1e00
.equ .Lwrite_data, 0x0000
.equ .Lwrite_mark, 0x0200
.equ .Lwrite_CRC, 0x0400
.equ .Lwrite_parameter, 0x0600
.equ .Lwrite_phase, 0x0800
.equ .Lwrite_setup, 0x0a00
.equ .Lwrite_mode0, 0x0c00
.equ .Lwrite_mode1, 0x0e00
.equ .Lread_data, 0x1000
.equ .Lread_mark, 0x1200
.equ .Lread_error, 0x1400
.equ .Lread_parameter, 0x1600
.equ .Lread_phase, 0x1800
.equ .Lread_setup, 0x1a00
.equ .Lread_status, 0x1c00
.equ .Lread_handshake, 0x1e00
.equ o_side, 0
.equ o_track, 1
.equ o_sector, 2
.equ o_size, 3
.equ o_crc0, 4
.equ o_crc1, 5
.equ .Lo_side, 0
.equ .Lo_track, 1
.equ .Lo_sector, 2
.equ .Lo_size, 3
.equ .Lo_crc0, 4
.equ .Lo_crc1, 5
.equ seek_time, 30000
.equ max_retry, 40
.equ sector_size, 512
.equ .Lseek_time, 30000
.equ .Lmax_retry, 40
.equ .Lsector_size, 512
.equ .Lmark_sequence_len, 4
.equ .Lmr_clear_fifo, 0x01
.equ .Lmr_action, 0x08
.equ .Lmr_write_action, 0x18
.equ .Lhr_crc_error, 0x02
.equ .Lhr_fifo_2bytes, 0x40
.equ .Lhr_fifo_1byte, 0x80
.Lmfm_mark_check:
/*
* This subroutine reads and validates a mark byte sequence.
* On entry, %a1 and %d4 shall hold the location and length (resp.)
* of the mark byte array.
* %a2 and %a3 shall hold the locations of the handshake and mark
* registers.
* Returns zero in %d1 for success.
*/
moveq #-1, %d1
subq #1, %d4
movew #.Lseek_time, %d2
5: tstb %a2@
dbmi %d2, 5b
bpl 6f
moveb %a3@, %d3
cmpb %a1@+, %d3
dbne %d4, 5b
bne 6f
moveq #0, %d1
6: rts
.global swim_read_sector_header
swim_read_sector_header:
link %a6, #0
moveml %d1-%d5/%a0-%a4,%sp@-
moveml %d1-%d5/%a0-%a5,%sp@-
movel %a6@(0x0c), %a4
bsr mfm_read_addrmark
moveml %sp@+, %d1-%d5/%a0-%a4
moveq #-1, %d0
bsr .Lmfm_read_header
moveml %sp@+, %d1-%d5/%a0-%a5
unlk %a6
rts
sector_address_mark:
.Lsector_address_mark:
.byte 0xa1, 0xa1, 0xa1, 0xfe
sector_data_mark:
.Lsector_data_mark:
.byte 0xa1, 0xa1, 0xa1, 0xfb
mfm_read_addrmark:
.Lmfm_read_header:
movel %a6@(0x08), %a3
lea %a3@(read_handshake), %a2
lea %a3@(read_mark), %a3
moveq #-1, %d0
movew #seek_time, %d2
lea %a3@(.Lread_handshake), %a2
lea %a3@(.Lread_data), %a5
lea %a3@(.Lread_mark), %a3
wait_header_init:
tstb %a3@(read_error - read_mark)
moveb #0x18, %a3@(write_mode0 - read_mark)
moveb #0x01, %a3@(write_mode1 - read_mark)
moveb #0x01, %a3@(write_mode0 - read_mark)
tstb %a3@(read_error - read_mark)
moveb #0x08, %a3@(write_mode1 - read_mark)
moveb #.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
moveb #.Lmr_clear_fifo, %a3@(.Lwrite_mode1 - .Lread_mark)
moveb #.Lmr_clear_fifo, %a3@(.Lwrite_mode0 - .Lread_mark)
tstb %a3@(.Lread_error - .Lread_mark)
moveb #.Lmr_action, %a3@(.Lwrite_mode1 - .Lread_mark)
lea sector_address_mark, %a0
moveq #3, %d1
lea .Lsector_address_mark, %a1
moveq #.Lmark_sequence_len, %d4
bsr .Lmfm_mark_check
tstl %d1
bne .Lsignal_nonyb
wait_addr_mark_byte:
/* read header */
moveq #.Lmax_retry, %d2
.Lamark0:
tstb %a2@
dbmi %d2, wait_addr_mark_byte
bpl header_exit
dbmi %d2, .Lamark0
bpl .Lsignal_nonyb
moveb %a3@, %d3
cmpb %a0@+, %d3
dbne %d1, wait_addr_mark_byte
bne wait_header_init
moveb %a5@, %a4@(.Lo_track)
moveq #max_retry, %d2
moveq #.Lmax_retry, %d2
.Lamark1:
tstb %a2@
dbmi %d2, .Lamark1
bpl .Lsignal_nonyb
amark0: tstb %a2@
dbmi %d2, amark0
bpl signal_nonyb
moveb %a5@, %a4@(.Lo_side)
moveb %a3@, %a4@(o_track)
moveq #.Lmax_retry, %d2
.Lamark2:
tstb %a2@
dbmi %d2, .Lamark2
bpl .Lsignal_nonyb
moveq #max_retry, %d2
moveb %a5@, %a4@(.Lo_sector)
amark1: tstb %a2@
dbmi %d2, amark1
bpl signal_nonyb
moveq #.Lmax_retry, %d2
.Lamark3:
tstb %a2@
dbmi %d2, .Lamark3
bpl .Lsignal_nonyb
moveb %a3@, %a4@(o_side)
moveb %a5@, %a4@(.Lo_size)
moveq #max_retry, %d2
moveq #.Lmax_retry, %d2
.Lcrc0:
tstb %a2@
dbmi %d2, .Lcrc0
bpl .Lsignal_nonyb
amark2: tstb %a2@
dbmi %d2, amark2
bpl signal_nonyb
moveb %a5@, %a4@(.Lo_crc0)
moveb %a3@, %a4@(o_sector)
moveq #.Lmax_retry, %d2
.Lcrc1:
tstb %a2@
dbmi %d2, .Lcrc1
bpl .Lsignal_nonyb
moveq #max_retry, %d2
moveb %a5@, %a4@(.Lo_crc1)
amark3: tstb %a2@
dbmi %d2, amark3
bpl signal_nonyb
moveb %a2@, %d5
andb #.Lhr_crc_error, %d5
bne .Lsignal_nonyb
moveb %a3@, %a4@(o_size)
moveq #max_retry, %d2
crc0: tstb %a2@
dbmi %d2, crc0
bpl signal_nonyb
moveb %a3@, %a4@(o_crc0)
moveq #max_retry, %d2
crc1: tstb %a2@
dbmi %d2, crc1
bpl signal_nonyb
moveb %a3@, %a4@(o_crc1)
tstb %a3@(read_error - read_mark)
header_exit:
moveq #0, %d0
moveb #0x18, %a3@(write_mode0 - read_mark)
rts
signal_nonyb:
moveq #-1, %d0
moveb #0x18, %a3@(write_mode0 - read_mark)
.Lsignal_nonyb:
moveb #.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
rts
.global swim_read_sector_data
@@ -150,94 +175,85 @@ swim_read_sector_data:
link %a6, #0
moveml %d1-%d5/%a0-%a5,%sp@-
movel %a6@(0x0c), %a4
bsr mfm_read_data
moveq #-1, %d0
bsr .Lmfm_read_data
moveml %sp@+, %d1-%d5/%a0-%a5
unlk %a6
rts
mfm_read_data:
.Lmfm_read_data:
movel %a6@(0x08), %a3
lea %a3@(read_handshake), %a2
lea %a3@(read_data), %a5
lea %a3@(read_mark), %a3
movew #seek_time, %d2
lea %a3@(.Lread_handshake), %a2
lea %a3@(.Lread_data), %a5
lea %a3@(.Lread_mark), %a3
wait_data_init:
tstb %a3@(read_error - read_mark)
moveb #0x18, %a3@(write_mode0 - read_mark)
moveb #0x01, %a3@(write_mode1 - read_mark)
moveb #0x01, %a3@(write_mode0 - read_mark)
tstb %a3@(read_error - read_mark)
moveb #0x08, %a3@(write_mode1 - read_mark)
moveb #.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
moveb #.Lmr_clear_fifo, %a3@(.Lwrite_mode1 - .Lread_mark)
moveb #.Lmr_clear_fifo, %a3@(.Lwrite_mode0 - .Lread_mark)
tstb %a3@(.Lread_error - .Lread_mark)
moveb #.Lmr_action, %a3@(.Lwrite_mode1 - .Lread_mark)
lea sector_data_mark, %a0
moveq #3, %d1
/* wait data address mark */
wait_data_mark_byte:
tstb %a2@
dbmi %d2, wait_data_mark_byte
bpl data_exit
moveb %a3@, %d3
cmpb %a0@+, %d3
dbne %d1, wait_data_mark_byte
bne wait_data_init
lea .Lsector_data_mark, %a1
moveq #.Lmark_sequence_len, %d4
bsr .Lmfm_mark_check
tstl %d1
bne .Ldata_exit
/* read data */
tstb %a3@(read_error - read_mark)
movel #sector_size-1, %d4 /* sector size */
read_new_data:
movew #max_retry, %d2
read_data_loop:
movel #.Lsector_size - 1, %d4 /* sector size */
movew #.Lmax_retry, %d2
.Lread_data_loop:
moveb %a2@, %d5
andb #0xc0, %d5
dbne %d2, read_data_loop
beq data_exit
moveb %a5@, %a4@+
andb #0x40, %d5
dbne %d4, read_new_data
beq exit_loop
moveb %a5@, %a4@+
dbra %d4, read_new_data
exit_loop:
andb #(.Lhr_fifo_1byte + .Lhr_fifo_2bytes), %d5
dbne %d2, .Lread_data_loop
beq .Ldata_exit
moveq #.Lmax_retry, %d2
moveb %a5@, %d3
bsr .Lmaybe_store
dbra %d4, 1f
bra .Ldata_crc0
1: andb #.Lhr_fifo_2bytes, %d5
beq .Lread_data_loop
moveb %a5@, %d3
bsr .Lmaybe_store
dbra %d4, .Lread_data_loop
/* read CRC */
movew #max_retry, %d2
data_crc0:
movew #.Lmax_retry, %d2
.Ldata_crc0:
tstb %a2@
dbmi %d2, data_crc0
bpl data_exit
dbmi %d2, .Ldata_crc0
bpl .Ldata_exit
moveb %a3@, %d5
moveq #max_retry, %d2
data_crc1:
moveb %a5@, %d5
moveq #.Lmax_retry, %d2
.Ldata_crc1:
tstb %a2@
dbmi %d2, data_crc1
bpl data_exit
dbmi %d2, .Ldata_crc1
bpl .Ldata_exit
moveb %a3@, %d5
moveb %a5@, %d5
tstb %a3@(read_error - read_mark)
moveb #0x18, %a3@(write_mode0 - read_mark)
moveb %a2@, %d5
andb #.Lhr_crc_error, %d5
bne .Ldata_exit
/* return number of bytes read */
movel #sector_size, %d0
movel #.Lsector_size, %d0
addw #1, %d4
subl %d4, %d0
.Ldata_exit:
moveb #.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
rts
data_exit:
moveb #0x18, %a3@(write_mode0 - read_mark)
moveq #-1, %d0
rts
.Lmaybe_store:
tstl %a4
beq 9f
moveb %d3, %a4@+
9: rts
drivers/block/ublk_drv.c
+208 -53
View File
@@ -321,6 +321,8 @@ struct ublk_device {
#define UB_STATE_OPEN 0
#define UB_STATE_USED 1
#define UB_STATE_DELETED 2
/* STOP_DEV canceled the server's commands, until its release */
#define UB_STATE_STOPPING 3
unsigned long state;
int ub_number;
@@ -334,6 +336,13 @@ struct ublk_device {
u16 nr_queue_ready;
bool unprivileged_daemons;
struct mutex cancel_mutex;
/* the open /dev/ublkcN, protected by cancel_mutex */
struct file *ch_file;
/*
* A cancel started in this FETCH round. Set by ublk_set_canceling(),
* cleared only by ublk_reset_ch_dev() when a new round starts. While
* it is set, no queue clears its ->canceling.
*/
bool canceling;
pid_t ublksrv_tgid;
struct delayed_work exit_work;
@@ -687,7 +696,7 @@ free_req:
if (!zone->len)
break;
ret = disk_report_zone(disk, zone, i, args);
ret = disk_report_zone(disk, zone, done_zones, args);
if (ret)
goto out;
@@ -810,6 +819,8 @@ ublk_batch_alloc_fcmd(struct io_uring_cmd *cmd)
if (fcmd) {
fcmd->cmd = cmd;
fcmd->buf_group = READ_ONCE(cmd->sqe->buf_index);
/* a cancel may look at it before it is linked */
INIT_LIST_HEAD(&fcmd->node);
}
return fcmd;
}
@@ -2399,6 +2410,9 @@ static int ublk_ch_open(struct inode *inode, struct file *filp)
return -EBUSY;
filp->private_data = ub;
ub->ublksrv_tgid = current->tgid;
mutex_lock(&ub->cancel_mutex);
ub->ch_file = filp;
mutex_unlock(&ub->cancel_mutex);
return 0;
}
@@ -2415,11 +2429,17 @@ static void ublk_reset_ch_dev(struct ublk_device *ub)
spin_unlock(&ubq->cancel_lock);
}
/* a new FETCH round starts, the queues stay canceling until ready */
mutex_lock(&ub->cancel_mutex);
ub->canceling = false;
mutex_unlock(&ub->cancel_mutex);
/* set to NULL, otherwise new tasks cannot mmap io_cmd_buf */
ub->mm = NULL;
ub->nr_queue_ready = 0;
ub->unprivileged_daemons = false;
ub->ublksrv_tgid = -1;
clear_bit(UB_STATE_STOPPING, &ub->state);
}
static struct gendisk *ublk_get_disk(struct ublk_device *ub)
@@ -2539,12 +2559,15 @@ static void ublk_ch_release_work_fn(struct work_struct *work)
}
/*
* disk isn't attached yet, either device isn't live, or it has
* been removed already, so we needn't to do anything
* No disk: the device isn't live, or it has been removed already.
* There are no requests to abort, but the round still has to be
* reset, so that a new server can fetch and start the device.
*/
disk = ublk_get_disk(ub);
if (!disk)
goto out;
if (!disk) {
mutex_lock(&ub->mutex);
goto reset;
}
/*
* All uring_cmd are done now, so abort any request outstanding to
@@ -2576,7 +2599,7 @@ static void ublk_ch_release_work_fn(struct work_struct *work)
/* double check after grabbing lock */
if (!ub->ub_disk)
goto unlock;
goto reset;
/*
* Transition the device to the nosrv state. What exactly this
@@ -2602,29 +2625,36 @@ static void ublk_ch_release_work_fn(struct work_struct *work)
WRITE_ONCE(ublk_get_queue(ub, i)->fail_io, true);
}
}
unlock:
reset:
/*
* All uring_cmd has been done now, reset device & ubq. Under
* ub->mutex, so START_DEV sees the round either ready or reset.
*/
ublk_reset_ch_dev(ub);
mutex_unlock(&ub->mutex);
ublk_put_disk(disk);
/* all uring_cmd has been done now, reset device & ubq */
ublk_reset_ch_dev(ub);
out:
clear_bit(UB_STATE_OPEN, &ub->state);
/* put the reference grabbed in ublk_ch_release() */
ublk_put_device(ub);
module_put(THIS_MODULE);
}
static int ublk_ch_release(struct inode *inode, struct file *filp)
{
struct ublk_device *ub = filp->private_data;
mutex_lock(&ub->cancel_mutex);
ub->ch_file = NULL;
mutex_unlock(&ub->cancel_mutex);
/*
* Grab ublk device reference, so it won't be gone until we are
* really released from work function.
*/
ublk_get_device(ub);
/* Keep the worker's callback and final device release code resident. */
__module_get(THIS_MODULE);
INIT_DELAYED_WORK(&ub->exit_work, ublk_ch_release_work_fn);
schedule_delayed_work(&ub->exit_work, 0);
return 0;
@@ -2732,9 +2762,11 @@ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
static void ublk_start_cancel(struct ublk_device *ub)
{
struct gendisk *disk = ublk_get_disk(ub);
struct gendisk *disk;
/* sync with ublk_ctrl_start_dev() publishing the disk */
mutex_lock(&ub->cancel_mutex);
disk = ublk_get_disk(ub);
if (ub->canceling)
goto out;
@@ -2790,7 +2822,8 @@ static void ublk_cancel_cmd(struct ublk_queue *ubq, u16 tag,
done = !!(io->flags & UBLK_IO_FLAG_CANCELED);
if (!done) {
io->flags |= UBLK_IO_FLAG_CANCELED;
cmd = io->cmd;
/* dependency ordered against smp_wmb() in ublk_prep_cancel() */
cmd = READ_ONCE(io->cmd);
io->cmd = NULL;
}
spin_unlock(&ubq->cancel_lock);
@@ -2879,6 +2912,7 @@ static void ublk_uring_cmd_cancel_fn(struct io_uring_cmd *cmd,
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct ublk_queue *ubq = pdu->ubq;
struct io_uring_cmd *cur;
struct task_struct *task;
struct ublk_io *io;
@@ -2895,7 +2929,9 @@ static void ublk_uring_cmd_cancel_fn(struct io_uring_cmd *cmd,
ublk_start_cancel(ubq->dev);
WARN_ON_ONCE(io->cmd != cmd);
/* NULL if a cancel from the control path took it meanwhile */
cur = READ_ONCE(io->cmd);
WARN_ON_ONCE(cur && cur != cmd);
ublk_cancel_cmd(ubq, pdu->tag, issue_flags);
}
@@ -3006,13 +3042,54 @@ static void ublk_stop_dev_unlocked(struct ublk_device *ub)
put_disk(disk);
}
static struct file *ublk_get_ch_file(struct ublk_device *ub)
{
struct file *file;
mutex_lock(&ub->cancel_mutex);
file = ub->ch_file;
if (file && !file_ref_get(&file->f_ref))
file = NULL;
mutex_unlock(&ub->cancel_mutex);
return file;
}
static void ublk_stop_dev(struct ublk_device *ub)
{
struct file *file;
/*
* FETCH, PREP and START_DEV take ub->mutex. If a server has
* /dev/ublkcN open, set STOPPING, which turns it away, and hold a
* reference on the file: the server's release, whose reset clears
* STOPPING and lets a new server attach, can't run before we drop it.
* So the cancel can run after the unlock and only meets this server's
* commands; it has to: io_uring_cmd_done() may take uring_lock, under
* which FETCH takes ub->mutex.
*/
mutex_lock(&ub->mutex);
ublk_stop_dev_unlocked(ub);
mutex_unlock(&ub->mutex);
cancel_work_sync(&ub->partition_scan_work);
file = ublk_get_ch_file(ub);
if (file) {
/*
* The server has to close /dev/ublkcN before this device can
* be started again: the reset in its release clears STOPPING.
*/
set_bit(UB_STATE_STOPPING, &ub->state);
/* for wake_up_var() below, see wake_up_bit() */
smp_mb__after_atomic();
}
mutex_unlock(&ub->mutex);
/* no server: nothing to cancel */
if (!file)
return;
/* wake a START_DEV waiting for the device to get ready */
wake_up_var(&ub->nr_queue_ready);
ublk_cancel_dev(ub);
__fput_sync(file);
}
static void ublk_reset_io_flags(struct ublk_queue *ubq, struct ublk_io *io)
@@ -3024,11 +3101,19 @@ static void ublk_reset_io_flags(struct ublk_queue *ubq, struct ublk_io *io)
}
/* reset per-queue io flags */
static void ublk_queue_reset_io_flags(struct ublk_queue *ubq)
static void ublk_queue_reset_io_flags(struct ublk_device *ub,
struct ublk_queue *ubq)
{
spin_lock(&ubq->cancel_lock);
ubq->canceling = false;
spin_unlock(&ubq->cancel_lock);
/*
* A cancel in this FETCH round took a command which still counts as
* ready, so the queue has to stay canceling. ub->canceling is set
* under cancel_mutex before any command is taken: either we see it
* here, or the cancel marks this queue again later.
*/
mutex_lock(&ub->cancel_mutex);
if (!ub->canceling)
ubq->canceling = false;
mutex_unlock(&ub->cancel_mutex);
ubq->fail_io = false;
ubq->force_abort = false;
}
@@ -3051,24 +3136,17 @@ static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id,
ub->nr_queue_ready++;
/*
* Reset queue flags as soon as this queue is ready.
* This clears the canceling flag, allowing batch FETCH commands
* to succeed during recovery without waiting for all queues.
* Reset queue flags as soon as this queue is ready. Unless
* this round saw a cancel, this clears the canceling flag,
* allowing batch FETCH commands to succeed during recovery
* without waiting for all queues.
*/
ublk_queue_reset_io_flags(ubq);
ublk_queue_reset_io_flags(ub, ubq);
}
/* Check if all queues are ready */
if (ublk_dev_ready(ub)) {
/*
* All queues ready - clear device-level canceling flag
* and wake ublk_dev_ready() waiters.
*/
mutex_lock(&ub->cancel_mutex);
ub->canceling = false;
mutex_unlock(&ub->cancel_mutex);
/* All queues ready - wake ublk_dev_ready() waiters */
if (ublk_dev_ready(ub))
wake_up_var(&ub->nr_queue_ready);
}
}
static inline int ublk_check_cmd_op(u32 cmd_op)
@@ -3152,6 +3230,12 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd)
return req;
}
/*
* Call before ublk_fill_io_cmd() publishes @cmd in io->cmd: a control-path
* cancel may complete any command found there, and io_uring_cmd_done() only
* takes it off the cancelable list if it is marked already. The handlers
* hold uring_lock, so marking takes no lock.
*/
static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
unsigned int issue_flags,
struct ublk_queue *ubq, u16 tag)
@@ -3165,6 +3249,8 @@ static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
pdu->ubq = ubq;
pdu->tag = tag;
io_uring_cmd_mark_cancelable(cmd, issue_flags);
/* pairs with the cancel loading cmd from io->cmd, then cmd->flags */
smp_wmb();
}
static void ublk_io_release(void *priv)
@@ -3269,6 +3355,9 @@ static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr)
static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
struct ublk_io *io, u16 q_id)
{
if (test_bit(UB_STATE_STOPPING, &ub->state))
return UBLK_IO_RES_ABORT;
/* UBLK_IO_FETCH_REQ is only allowed before dev is setup */
if (ublk_dev_ready(ub))
return -EBUSY;
@@ -3412,11 +3501,11 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
ret = ublk_check_fetch_buf(ub, addr);
if (ret)
goto out;
/* before ublk_fetch() publishes io->cmd, see ublk_prep_cancel() */
ublk_prep_cancel(cmd, issue_flags, ubq, tag);
ret = ublk_fetch(cmd, ub, io, addr, q_id);
if (ret)
goto out;
ublk_prep_cancel(cmd, issue_flags, ubq, tag);
goto out_done;
return -EIOCBQUEUED;
}
@@ -3460,6 +3549,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
if (ret)
goto out;
io->res = result;
ublk_prep_cancel(cmd, issue_flags, ubq, tag);
req = ublk_fill_io_cmd(io, cmd);
ublk_apply_io_buf(ub, io, cmd, addr, &auto_buf, &buf_idx);
if (buf_idx != UBLK_INVALID_BUF_IDX)
@@ -3478,19 +3568,24 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
* uring_cmd active first and prepare for handling new requeued
* request
*/
ublk_prep_cancel(cmd, issue_flags, ubq, tag);
req = ublk_fill_io_cmd(io, cmd);
io->buf.addr = addr;
if (likely(ublk_get_data(ubq, io, req))) {
__ublk_prep_compl_io_cmd(io, req);
return UBLK_IO_RES_OK;
ret = UBLK_IO_RES_OK;
goto out_done;
}
break;
default:
goto out;
}
ublk_prep_cancel(cmd, issue_flags, ubq, tag);
return -EIOCBQUEUED;
out_done:
/* marked cancelable: complete through io_uring_cmd_done() */
io_uring_cmd_done(cmd, ret, issue_flags);
return -EIOCBQUEUED;
out:
pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
__func__, cmd_op, tag, ret, io ? io->flags : 0);
@@ -3889,6 +3984,15 @@ static int ublk_batch_attach(struct ublk_queue *ubq,
bool free = false;
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(data->cmd);
/*
* Mark it cancelable before linking it into fcmd_head, where a cancel
* from the control path can take and complete it: see
* ublk_prep_cancel(). evts_lock orders the mark before the link.
*/
pdu->ubq = ubq;
pdu->fcmd = fcmd;
io_uring_cmd_mark_cancelable(fcmd->cmd, data->issue_flags);
spin_lock(&ubq->evts_lock);
if (unlikely(ubq->force_abort || ubq->canceling)) {
free = true;
@@ -3899,14 +4003,12 @@ static int ublk_batch_attach(struct ublk_queue *ubq,
spin_unlock(&ubq->evts_lock);
if (unlikely(free)) {
/* off the cancelable list first, then nothing can see fcmd */
io_uring_cmd_done(data->cmd, -ENODEV, data->issue_flags);
ublk_batch_free_fcmd(fcmd);
return -ENODEV;
return -EIOCBQUEUED;
}
pdu->ubq = ubq;
pdu->fcmd = fcmd;
io_uring_cmd_mark_cancelable(fcmd->cmd, data->issue_flags);
if (!new_fcmd)
goto out;
@@ -3914,9 +4016,12 @@ static int ublk_batch_attach(struct ublk_queue *ubq,
* If the two fetch commands are originated from same io_ring_ctx,
* run batch dispatch directly. Otherwise, schedule task work for
* doing it.
*
* Use data->cmd, not fcmd->cmd: once fcmd is linked and not active,
* a cancel from the control path may complete and free it.
*/
if (io_uring_cmd_ctx_handle(new_fcmd->cmd) ==
io_uring_cmd_ctx_handle(fcmd->cmd)) {
io_uring_cmd_ctx_handle(data->cmd)) {
data->cmd = new_fcmd->cmd;
ublk_batch_dispatch(ubq, data, new_fcmd);
} else {
@@ -4389,6 +4494,7 @@ static int ublk_add_tag_set(struct ublk_device *ub)
ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
ub->tag_set.queue_depth = ub->dev_info.queue_depth;
ub->tag_set.numa_node = NUMA_NO_NODE;
ub->tag_set.flags = BLK_MQ_F_NO_SCHED_BY_DEFAULT;
ub->tag_set.driver_data = ub;
return blk_mq_alloc_tag_set(&ub->tag_set);
}
@@ -4431,21 +4537,27 @@ static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid)
return ub->ublksrv_tgid == ublksrv_pid;
}
static bool ublk_dev_ready_or_stopping(const struct ublk_device *ub)
{
return ublk_dev_ready(ub) || test_bit(UB_STATE_STOPPING, &ub->state);
}
/*
* Wait until all queues have fetched their I/O commands, and return with
* ub->mutex held and readiness guaranteed: then every queue's ->canceling
* is cleared. Ready may regress between wakeup and mutex_lock() (F_BATCH
* UNPREP, daemon death), so re-check it under the mutex and wait again.
* Wait until all queues have fetched their I/O commands, or STOP_DEV set
* UB_STATE_STOPPING, and return with ub->mutex held. The queues stay
* canceling if this round saw a cancel, see ublk_queue_reset_io_flags().
* Ready may regress between wakeup and mutex_lock() (F_BATCH UNPREP,
* daemon death), so re-check it under the mutex and wait again.
*/
static int ublk_wait_dev_ready_and_lock(struct ublk_device *ub)
{
while (true) {
if (wait_var_event_interruptible(&ub->nr_queue_ready,
ublk_dev_ready(ub)))
ublk_dev_ready_or_stopping(ub)))
return -EINTR;
mutex_lock(&ub->mutex);
if (ublk_dev_ready(ub))
if (ublk_dev_ready_or_stopping(ub))
return 0;
mutex_unlock(&ub->mutex);
}
@@ -4469,6 +4581,7 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub,
.dma_alignment = 3,
};
struct gendisk *disk;
bool canceled;
int ret = -EINVAL;
if (ublksrv_pid <= 0)
@@ -4545,6 +4658,10 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub,
ret = -EEXIST;
goto out_unlock;
}
if (test_bit(UB_STATE_STOPPING, &ub->state)) {
ret = -EBUSY;
goto out_unlock;
}
disk = blk_mq_alloc_disk(&ub->tag_set, &lim, NULL);
if (IS_ERR(disk)) {
@@ -4555,8 +4672,24 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub,
disk->fops = &ub_fops;
disk->private_data = ub;
/*
* A command of this FETCH round was canceled and can't be fetched
* again, don't bring up a disk over it. Check and publish the disk
* in one cancel_mutex section: either this sees ub->canceling, or
* ublk_start_cancel() sees the disk and quiesces it before marking
* the queues.
*/
mutex_lock(&ub->cancel_mutex);
canceled = ub->canceling;
if (!canceled)
ub->ub_disk = disk;
mutex_unlock(&ub->cancel_mutex);
if (canceled) {
put_disk(disk);
ret = -EBUSY;
goto out_unlock;
}
ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
ub->ub_disk = disk;
ublk_apply_params(ub);
@@ -4901,10 +5034,15 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub, bool wait)
set_bit(UB_STATE_DELETED, &ub->state);
}
/* Mark the reference as consumed */
mutex_unlock(&ublk_ctl_mutex);
/*
* Drop the reference outside ublk_ctl_mutex: if it is the last one,
* the release frees the tag set, which waits for an SRCU grace period,
* and holding the mutex would serialize concurrent deletions on it.
*/
*p_ub = NULL;
ublk_put_device(ub);
mutex_unlock(&ublk_ctl_mutex);
/*
* Wait until the idr is removed, then it can be reused after
@@ -5128,6 +5266,7 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub,
const struct ublksrv_ctrl_cmd *header)
{
int ublksrv_pid = (int)header->data[0];
bool canceled;
int ret = -EINVAL;
pr_devel("%s: Waiting for all FETCH_REQs, dev id %d...\n", __func__,
@@ -5151,6 +5290,20 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub,
ret = -EBUSY;
goto out_unlock;
}
/*
* As in ublk_ctrl_start_dev(), a canceled command can't be fetched
* again. Best effort: the disk exists here, and a cancel after this
* check is the ordinary death of the new server, which
* ublk_start_cancel() handles by quiescing and marking.
*/
mutex_lock(&ub->cancel_mutex);
canceled = ub->canceling;
mutex_unlock(&ub->cancel_mutex);
if (canceled) {
ret = -EBUSY;
goto out_unlock;
}
ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
ub->dev_info.state = UBLK_S_DEV_LIVE;
pr_devel("%s: new ublksrv_pid %d, dev id %d\n",
@@ -5628,6 +5781,8 @@ static bool ublk_try_buf_match(struct ublk_device *ub,
unsigned long expected_offset = 0;
bool first = true;
guard(rcu)();
rq_for_each_bvec(bv, rq, iter) {
unsigned long pfn = page_to_pfn(bv.bv_page);
unsigned long end_pfn = pfn +
drivers/block/zloop.c
+24 -14
View File
@@ -318,22 +318,23 @@ static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no)
ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0);
if (ret < 0) {
pr_err("Failed to get zone %u file stat (err=%d)\n",
zone_no, ret);
pr_err("%pg: Failed to get zone %u file stat (err=%d)\n",
zlo->disk->part0, zone_no, ret);
set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
return ret;
}
file_sectors = stat.size >> SECTOR_SHIFT;
if (file_sectors > zlo->zone_capacity) {
pr_err("Zone %u file too large (%llu sectors > %llu)\n",
zone_no, file_sectors, zlo->zone_capacity);
pr_err("%pg: Zone %u file too large (%llu sectors > %llu)\n",
zlo->disk->part0, zone_no, file_sectors,
zlo->zone_capacity);
return -EINVAL;
}
if (!IS_ALIGNED(stat.size, zlo->block_size)) {
pr_err("Zone %u file size (%llu) not aligned to block size %u\n",
zone_no, stat.size, zlo->block_size);
pr_err("%pg: Zone %u file size (%llu) not aligned to block size %u\n",
zlo->disk->part0, zone_no, stat.size, zlo->block_size);
return -EINVAL;
}
@@ -603,8 +604,15 @@ static int zloop_seq_write_prep(struct zloop_cmd *cmd)
}
} else {
if (cmd->sector != zone->wp) {
pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n",
zone_no, cmd->sector, zone->wp);
if (zone->wp == ULLONG_MAX)
pr_err("%pg: Zone %u: write to %s zone: sect %llu\n",
zlo->disk->part0, zone_no,
blk_zone_cond_str(zone->cond),
cmd->sector);
else
pr_err("%pg: Zone %u: unaligned write: sect %llu, wp %llu\n",
zlo->disk->part0, zone_no, cmd->sector,
zone->wp);
ret = -EIO;
goto out_unlock;
}
@@ -832,8 +840,9 @@ static void zloop_complete_rq(struct request *rq)
switch (req_op(rq)) {
case REQ_OP_READ:
if (cmd->ret < 0)
pr_err("Zone %u: failed read sector %llu, %llu sectors\n",
zone_no, cmd->sector, cmd->nr_sectors);
pr_err("%pg: Zone %u: failed read sector %llu, %llu sectors\n",
zlo->disk->part0, zone_no, cmd->sector,
cmd->nr_sectors);
/* Short read. Zero out the unread tail beyond cmd->ret. */
if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq))
@@ -842,14 +851,15 @@ static void zloop_complete_rq(struct request *rq)
case REQ_OP_WRITE:
case REQ_OP_ZONE_APPEND:
if (cmd->ret < 0)
pr_err("Zone %u: failed %swrite sector %llu, %llu sectors\n",
zone_no,
pr_err("%pg: Zone %u: failed %swrite sector %llu, %llu sectors\n",
zlo->disk->part0, zone_no,
req_op(rq) == REQ_OP_WRITE ? "" : "append ",
cmd->sector, cmd->nr_sectors);
if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) {
pr_err("Zone %u: partial write %ld/%u B\n",
zone_no, cmd->ret, blk_rq_bytes(rq));
pr_err("%pg: Zone %u: partial write %ld/%u B\n",
zlo->disk->part0, zone_no, cmd->ret,
blk_rq_bytes(rq));
cmd->ret = -EIO;
}
drivers/nvme/host/apple.c
+35 -4
View File
@@ -42,7 +42,11 @@
#define APPLE_ANS_ACQ_DB 0x1004
#define APPLE_ANS_IOCQ_DB 0x100c
#define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210
#define APPLE_ANS_T8132_IOQ_CMDS 0x1200
#define APPLE_ANS_T8132_IOQ_CQES 0x1208
#define APPLE_ANS_T8132_IOQ_SIZE 0x1210
#define APPLE_ANS_T8103_MAX_PEND_CMDS_CTRL 0x1210
#define APPLE_ANS_BOOT_STATUS 0x1300
#define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55
@@ -184,6 +188,7 @@ struct apple_nvme_iod {
struct apple_nvme_hw {
bool has_lsq_nvmmu;
bool needs_ioq_register;
u32 max_queue_depth;
};
@@ -1133,9 +1138,16 @@ static void apple_nvme_reset_work(struct work_struct *work)
anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL);
/* Allow as many pending command as possible for both queues */
writel(anv->hw->max_queue_depth
| (anv->hw->max_queue_depth << 16), anv->mmio_nvme
+ APPLE_ANS_MAX_PEND_CMDS_CTRL);
if (!anv->hw->needs_ioq_register) {
/*
* Since T8132, this is replaced by IOQ_SIZE which is
* located at the same offset and is handled with the
* IO queue setup further below.
*/
writel(anv->hw->max_queue_depth
| (anv->hw->max_queue_depth << 16), anv->mmio_nvme
+ APPLE_ANS_T8103_MAX_PEND_CMDS_CTRL);
}
/* Setup the NVMMU for the maximum admin and IO queue depth */
writel(anv->hw->max_queue_depth - 1,
@@ -1190,6 +1202,16 @@ static void apple_nvme_reset_work(struct work_struct *work)
if (ret)
goto out_remove_cq;
if (anv->hw->needs_ioq_register) {
writeq(anv->ioq.cq_dma_addr,
anv->mmio_nvme + APPLE_ANS_T8132_IOQ_CQES);
writeq(anv->ioq.sq_dma_addr,
anv->mmio_nvme + APPLE_ANS_T8132_IOQ_CMDS);
writel((anv->hw->max_queue_depth - 1)
| ((anv->hw->max_queue_depth - 1) << 16), anv->mmio_nvme
+ APPLE_ANS_T8132_IOQ_SIZE);
}
apple_nvme_init_queue(&anv->ioq);
nr_io_queues = 1;
ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues);
@@ -1699,17 +1721,26 @@ static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend,
static const struct apple_nvme_hw apple_nvme_t8015_hw = {
.has_lsq_nvmmu = false,
.needs_ioq_register = false,
.max_queue_depth = 16,
};
static const struct apple_nvme_hw apple_nvme_t8103_hw = {
.has_lsq_nvmmu = true,
.needs_ioq_register = false,
.max_queue_depth = 64,
};
static const struct apple_nvme_hw apple_nvme_t8132_hw = {
.has_lsq_nvmmu = true,
.needs_ioq_register = true,
.max_queue_depth = 64,
};
static const struct of_device_id apple_nvme_of_match[] = {
{ .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw },
{ .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw },
{ .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw },
{ .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw },
{},
};
drivers/nvme/host/auth.c
+14 -1
View File
@@ -358,8 +358,21 @@ static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
nvme_auth_hmac_name(chap->hash_id),
nvme_auth_dhgroup_name(chap->dhgroup_id));
if (!data->rvalid)
if (!data->rvalid) {
/*
* The controller did not return a response. If we asked it to
* authenticate itself the session must not be used, otherwise
* mutual authentication would silently degrade to one-way.
*/
if (chap->bi_directional) {
dev_warn(ctrl->device,
"qid %d: controller did not authenticate\n",
chap->qid);
chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
return -ECONNREFUSED;
}
return 0;
}
/* Validate controller response */
if (crypto_memneq(chap->response, data->rval, data->hl)) {
drivers/nvme/host/core.c
+10 -5
View File
@@ -2273,13 +2273,18 @@ static int nvme_query_fdp_granularity(struct nvme_ctrl *ctrl,
}
size = le32_to_cpu(hdr.sze);
if (size < sizeof(*h) + sizeof(*desc)) {
dev_warn(ctrl->device, "FDP config size too small:%zu\n",
size);
return 0;
}
if (size > PAGE_SIZE * MAX_ORDER_NR_PAGES) {
dev_warn(ctrl->device, "FDP config size too large:%zu\n",
size);
return 0;
}
h = kvmalloc(size, GFP_KERNEL);
h = kvzalloc(size, GFP_KERNEL);
if (!h)
return -ENOMEM;
@@ -2305,9 +2310,9 @@ static int nvme_query_fdp_granularity(struct nvme_ctrl *ctrl,
desc = log;
end = log + size - sizeof(*h);
for (i = 0; i < fdp_idx; i++) {
u16 dsze = le16_to_cpu(desc->dsze);
u16 dsze = get_unaligned_le16(&desc->dsze);
if (!dsze || log + dsze > end) {
if (dsze < sizeof(*desc) || log + dsze + sizeof(*desc) > end) {
dev_warn(ctrl->device,
"FDP invalid config descriptor at index %d\n", i);
ret = 0;
@@ -2317,13 +2322,13 @@ static int nvme_query_fdp_granularity(struct nvme_ctrl *ctrl,
desc = log;
}
if (le32_to_cpu(desc->nrg) > 1) {
if (get_unaligned_le32(&desc->nrg) > 1) {
dev_warn(ctrl->device, "FDP NRG > 1 not supported\n");
ret = 0;
goto out;
}
info->runs = le64_to_cpu(desc->runs);
info->runs = get_unaligned_le64(&desc->runs);
out:
kvfree(h);
return ret;
drivers/nvme/host/ioctl.c
+1 -1
View File
@@ -343,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
c.common.cdw14 = cpu_to_le32(cmd.cdw14);
c.common.cdw15 = cpu_to_le32(cmd.cdw15);
if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write))
if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write))
return -EACCES;
if (cmd.timeout_ms)
drivers/nvme/host/multipath.c
+136 -19
View File
@@ -307,7 +307,14 @@ int nvme_mpath_revalidate_zones(struct nvme_ns_head *head)
}
#endif /* CONFIG_BLK_DEV_ZONED */
static bool nvme_path_is_disabled(struct nvme_ns *ns)
enum nvme_path_state {
NVME_PATH_ENABLED,
NVME_PATH_DISABLED_CTRL_DOWN,
NVME_PATH_DISABLED_ANA_PENDING,
NVME_PATH_DISABLED_NS_NOT_READY,
};
static enum nvme_path_state nvme_path_get_state(struct nvme_ns *ns)
{
enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);
@@ -317,11 +324,17 @@ static bool nvme_path_is_disabled(struct nvme_ns *ns)
* Otherwise it will fail immediately and return to the requeue list.
*/
if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING)
return true;
if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
!test_bit(NVME_NS_READY, &ns->flags))
return true;
return false;
return NVME_PATH_DISABLED_CTRL_DOWN;
if (test_bit(NVME_NS_ANA_PENDING, &ns->flags))
return NVME_PATH_DISABLED_ANA_PENDING;
if (!test_bit(NVME_NS_READY, &ns->flags))
return NVME_PATH_DISABLED_NS_NOT_READY;
return NVME_PATH_ENABLED;
}
static bool nvme_path_is_disabled(struct nvme_ns *ns)
{
return nvme_path_get_state(ns) != NVME_PATH_ENABLED;
}
static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
@@ -496,9 +509,17 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
}
}
static inline bool nvme_state_is_live(enum nvme_ana_state state)
{
return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
}
static bool nvme_available_path(struct nvme_ns_head *head)
__must_hold_shared(&head->srcu)
{
bool fail_on_ctrl_loss = test_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
&head->flags);
bool failfast = false;
struct nvme_ns *ns;
if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags))
@@ -506,18 +527,30 @@ static bool nvme_available_path(struct nvme_ns_head *head)
list_for_each_entry_srcu(ns, &head->list, siblings,
srcu_read_lock_held(&head->srcu)) {
if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) {
failfast = true;
continue;
}
switch (nvme_ctrl_state(ns->ctrl)) {
case NVME_CTRL_LIVE:
case NVME_CTRL_RESETTING:
case NVME_CTRL_CONNECTING:
if (!nvme_state_is_live(ns->ana_state) &&
ns->ana_state != NVME_ANA_CHANGE)
continue;
return true;
case NVME_CTRL_RESETTING:
return true;
case NVME_CTRL_CONNECTING:
if (!fail_on_ctrl_loss)
return true;
continue;
default:
break;
}
}
if (failfast || fail_on_ctrl_loss)
return false;
/*
* If "head->delayed_removal_secs" is configured (i.e., non-zero), do
* not immediately fail I/O. Instead, requeue the I/O for the configured
@@ -871,11 +904,6 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
return 0;
}
static inline bool nvme_state_is_live(enum nvme_ana_state state)
{
return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
}
static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
struct nvme_ns *ns)
{
@@ -910,8 +938,10 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
* is not live but still create the sysfs link to this path from
* head node if head node of the path has already come alive.
*/
if (test_bit(NVME_NSHEAD_DISK_LIVE, &ns->head->flags))
if (test_bit(NVME_NSHEAD_DISK_LIVE, &ns->head->flags)) {
nvme_mpath_add_sysfs_link(ns->head);
kblockd_schedule_work(&ns->head->requeue_work);
}
}
}
@@ -1038,6 +1068,10 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
struct nvme_subsystem *subsys =
container_of(dev, struct nvme_subsystem, dev);
/* A discovery subsystem has no namespaces and hence no I/O paths */
if (subsys->subtype == NVME_NQN_DISC)
return sysfs_emit(buf, "none\n");
return sysfs_emit(buf, "%s\n",
nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
}
@@ -1072,6 +1106,9 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
container_of(dev, struct nvme_subsystem, dev);
int policy;
if (subsys->subtype == NVME_NQN_DISC)
return -EOPNOTSUPP;
policy = nvme_iopolicy_parse(buf);
if (policy < 0)
return policy;
@@ -1082,6 +1119,26 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
static ssize_t iopolicies_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct nvme_subsystem *subsys =
container_of(dev, struct nvme_subsystem, dev);
int i, len = 0;
if (subsys->subtype == NVME_NQN_DISC)
return sysfs_emit(buf, "none\n");
for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++)
len += sysfs_emit_at(buf, len, "%s%s", i ? " " : "",
nvme_iopolicy_names[i]);
len += sysfs_emit_at(buf, len, "\n");
return len;
}
struct device_attribute subsys_attr_iopolicies = __ATTR_RO(iopolicies);
static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
@@ -1110,6 +1167,28 @@ static ssize_t queue_depth_show(struct device *dev,
}
DEVICE_ATTR_RO(queue_depth);
static const char * const nvme_path_state_names[] = {
[NVME_PATH_ENABLED] = "enabled",
[NVME_PATH_DISABLED_CTRL_DOWN] = "ctrl-down",
[NVME_PATH_DISABLED_ANA_PENDING] = "ana-pending",
[NVME_PATH_DISABLED_NS_NOT_READY] = "ns-not-ready",
};
static ssize_t path_state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
enum nvme_path_state state = nvme_path_get_state(ns);
const char *name = "unknown";
if (state < ARRAY_SIZE(nvme_path_state_names) &&
nvme_path_state_names[state])
name = nvme_path_state_names[state];
return sysfs_emit(buf, "%s\n", name);
}
DEVICE_ATTR_RO(path_state);
static ssize_t numa_nodes_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
@@ -1164,10 +1243,7 @@ static ssize_t delayed_removal_secs_store(struct device *dev,
mutex_lock(&head->subsys->lock);
head->delayed_removal_secs = sec;
if (sec)
set_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags);
else
clear_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags);
assign_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags, sec);
mutex_unlock(&head->subsys->lock);
/*
* Ensure that update to NVME_NSHEAD_QUEUE_IF_NO_PATH is seen
@@ -1180,6 +1256,47 @@ static ssize_t delayed_removal_secs_store(struct device *dev,
DEVICE_ATTR_RW(delayed_removal_secs);
static ssize_t fail_on_ctrl_loss_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct gendisk *disk = dev_to_disk(dev);
struct nvme_ns_head *head = disk->private_data;
return sysfs_emit(buf, test_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
&head->flags) ? "on\n" : "off\n");
}
static ssize_t fail_on_ctrl_loss_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct gendisk *disk = dev_to_disk(dev);
struct nvme_ns_head *head = disk->private_data;
bool enable;
int ret;
ret = kstrtobool(buf, &enable);
if (ret < 0)
return ret;
if (enable) {
if (test_and_set_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS, &head->flags))
return count;
} else {
if (!test_and_clear_bit(NVME_NSHEAD_FAIL_ON_CTRL_LOSS,
&head->flags))
return count;
}
synchronize_srcu(&head->srcu);
if (enable)
kblockd_schedule_work(&head->requeue_work);
return count;
}
DEVICE_ATTR_RW(fail_on_ctrl_loss);
static ssize_t multipath_failover_count_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
drivers/nvme/host/nvme.h
+4
View File
@@ -590,6 +590,7 @@ struct nvme_ns_head {
#define NVME_NSHEAD_DISK_LIVE 0
#define NVME_NSHEAD_QUEUE_IF_NO_PATH 1
#define NVME_NSHEAD_CDEV_LIVE 2
#define NVME_NSHEAD_FAIL_ON_CTRL_LOSS 3
struct nvme_ns __rcu_guarded *current_path[];
#endif
};
@@ -1096,11 +1097,14 @@ extern struct device_attribute dev_attr_ana_grpid;
extern struct device_attribute dev_attr_ana_state;
extern struct device_attribute dev_attr_queue_depth;
extern struct device_attribute dev_attr_numa_nodes;
extern struct device_attribute dev_attr_path_state;
extern struct device_attribute dev_attr_delayed_removal_secs;
extern struct device_attribute dev_attr_fail_on_ctrl_loss;
extern struct device_attribute dev_attr_multipath_failover_count;
extern struct device_attribute dev_attr_io_requeue_no_usable_path_count;
extern struct device_attribute dev_attr_io_fail_no_available_path_count;
extern struct device_attribute subsys_attr_iopolicy;
extern struct device_attribute subsys_attr_iopolicies;
static inline bool nvme_disk_is_ns_head(struct gendisk *disk)
{
drivers/nvme/host/pci.c
+1 -1
View File
@@ -3747,7 +3747,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
*/
dev->ctrl.max_hw_sectors = min_t(u32,
NVME_MAX_BYTES >> SECTOR_SHIFT,
dma_max_opt_mapping_size(&pdev->dev) >> SECTOR_SHIFT);
dma_max_mapping_size(&pdev->dev) >> SECTOR_SHIFT);
dev->ctrl.max_segments = NVME_MAX_SEGS;
dev->ctrl.max_integrity_segments = 1;
return dev;
drivers/nvme/host/rdma.c
+1
View File
@@ -1748,6 +1748,7 @@ static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
"Bogus remote invalidation for rkey %#x\n",
req->mr ? req->mr->rkey : 0);
nvme_rdma_error_recovery(queue->ctrl);
return;
}
} else if (req->mr) {
int ret;
drivers/nvme/host/sysfs.c
+13 -10
View File
@@ -263,7 +263,9 @@ static struct attribute *nvme_ns_attrs[] = {
&dev_attr_ana_state.attr,
&dev_attr_queue_depth.attr,
&dev_attr_numa_nodes.attr,
&dev_attr_path_state.attr,
&dev_attr_delayed_removal_secs.attr,
&dev_attr_fail_on_ctrl_loss.attr,
#endif
&dev_attr_io_passthru_err_log_enabled.attr,
NULL,
@@ -296,11 +298,13 @@ static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
return 0;
}
if (a == &dev_attr_queue_depth.attr || a == &dev_attr_numa_nodes.attr) {
if (a == &dev_attr_queue_depth.attr || a == &dev_attr_numa_nodes.attr ||
a == &dev_attr_path_state.attr) {
if (nvme_disk_is_ns_head(dev_to_disk(dev)))
return 0;
}
if (a == &dev_attr_delayed_removal_secs.attr) {
if (a == &dev_attr_delayed_removal_secs.attr ||
a == &dev_attr_fail_on_ctrl_loss.attr) {
struct gendisk *disk = dev_to_disk(dev);
if (!nvme_disk_is_ns_head(disk))
@@ -809,7 +813,7 @@ static DEVICE_ATTR(io_timeout, S_IRUGO | S_IWUSR,
nvme_io_timeout_show, nvme_io_timeout_store);
#ifdef CONFIG_NVME_HOST_AUTH
static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
static ssize_t dhchap_secret_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
@@ -820,7 +824,7 @@ static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
return sysfs_emit(buf, "%s\n", opts->dhchap_secret);
}
static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
static ssize_t dhchap_secret_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
@@ -864,10 +868,9 @@ static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
return count;
}
static DEVICE_ATTR(dhchap_secret, S_IRUGO | S_IWUSR,
nvme_ctrl_dhchap_secret_show, nvme_ctrl_dhchap_secret_store);
static DEVICE_ATTR_ADMIN_RW(dhchap_secret);
static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
static ssize_t dhchap_ctrl_secret_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
@@ -878,7 +881,7 @@ static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
return sysfs_emit(buf, "%s\n", opts->dhchap_ctrl_secret);
}
static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
static ssize_t dhchap_ctrl_secret_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
@@ -922,8 +925,7 @@ static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
return count;
}
static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR,
nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store);
static DEVICE_ATTR_ADMIN_RW(dhchap_ctrl_secret);
#endif
static struct attribute *nvme_dev_attrs[] = {
@@ -1298,6 +1300,7 @@ static struct attribute *nvme_subsys_attrs[] = {
&subsys_attr_subsystype.attr,
#ifdef CONFIG_NVME_MULTIPATH
&subsys_attr_iopolicy.attr,
&subsys_attr_iopolicies.attr,
#endif
NULL,
};
drivers/nvme/host/tcp.c
+96 -43
View File
@@ -7,6 +7,7 @@
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/async.h>
#include <linux/err.h>
#include <linux/crc32.h>
#include <linux/nvme-tcp.h>
@@ -54,7 +55,8 @@ MODULE_PARM_DESC(tls_handshake_timeout,
"nvme TLS handshake timeout in seconds (default 10)");
#endif
static atomic_t nvme_tcp_cpu_queues[NR_CPUS];
static int nvme_tcp_cpu_queues[NR_CPUS];
static DEFINE_SPINLOCK(nvme_tcp_cpu_queues_lock);
enum nvme_tcp_send_state {
NVME_TCP_SEND_CMD_PDU = 0,
@@ -149,6 +151,12 @@ struct nvme_tcp_queue {
static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
static LIST_HEAD_GUARDED(nvme_tcp_ctrl_list, nvme_tcp_ctrl_mutex);
struct nvme_tcp_queue_setup_ctx {
struct nvme_ctrl *ctrl;
int qid;
int *err;
};
struct nvme_tcp_ctrl {
/* read only in the hot path */
struct nvme_tcp_queue *queues;
@@ -166,6 +174,7 @@ struct nvme_tcp_ctrl {
struct delayed_work connect_work;
struct nvme_tcp_request async_req;
u32 io_queues[HCTX_MAX_TYPES];
struct net *net;
};
static struct workqueue_struct *nvme_tcp_wq;
@@ -1725,9 +1734,10 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
goto out;
/* Search for the least used cpu from the mq_map */
spin_lock(&nvme_tcp_cpu_queues_lock);
io_cpu = WORK_CPU_UNBOUND;
for_each_online_cpu(cpu) {
int num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]);
int num_queues = nvme_tcp_cpu_queues[cpu];
if (mq_map[cpu] != qid)
continue;
@@ -1738,9 +1748,10 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
}
if (io_cpu != WORK_CPU_UNBOUND) {
queue->io_cpu = io_cpu;
atomic_inc(&nvme_tcp_cpu_queues[io_cpu]);
nvme_tcp_cpu_queues[io_cpu]++;
set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);
}
spin_unlock(&nvme_tcp_cpu_queues_lock);
out:
dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
qid, queue->io_cpu);
@@ -1853,7 +1864,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
queue->cmnd_capsule_len = sizeof(struct nvme_command) +
NVME_TCP_ADMIN_CCSZ;
ret = sock_create_kern(current->nsproxy->net_ns,
ret = sock_create_kern(ctrl->net,
ctrl->addr.ss_family, SOCK_STREAM,
IPPROTO_TCP, &queue->sock);
if (ret) {
@@ -2009,8 +2020,11 @@ static void nvme_tcp_stop_queue_nowait(struct nvme_ctrl *nctrl, int qid)
if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return;
if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);
if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags)) {
spin_lock(&nvme_tcp_cpu_queues_lock);
nvme_tcp_cpu_queues[queue->io_cpu]--;
spin_unlock(&nvme_tcp_cpu_queues_lock);
}
mutex_lock(&queue->queue_lock);
if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
@@ -2117,25 +2131,6 @@ static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
nvme_tcp_wait_queue(ctrl, i);
}
static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
int first, int last)
{
int i, ret;
for (i = first; i < last; i++) {
ret = nvme_tcp_start_queue(ctrl, i);
if (ret)
goto out_stop_queues;
}
return 0;
out_stop_queues:
for (i--; i >= first; i--)
nvme_tcp_stop_queue(ctrl, i);
return ret;
}
static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
{
int ret;
@@ -2170,10 +2165,8 @@ out_free_queue:
return ret;
}
static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
static int nvme_tcp_tls_check_psk(struct nvme_ctrl *ctrl)
{
int i, ret;
if (nvme_tcp_tls_configured(ctrl)) {
if (ctrl->opts->concat) {
/*
@@ -2195,23 +2188,75 @@ static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
}
}
for (i = 1; i < ctrl->queue_count; i++) {
ret = nvme_tcp_alloc_queue(ctrl, i,
ctrl->tls_pskid);
if (ret)
goto out_free_queues;
return 0;
}
static void nvme_tcp_setup_queue_async(void *data, async_cookie_t cookie)
{
struct nvme_tcp_queue_setup_ctx *ctx = data;
struct nvme_ctrl *ctrl = ctx->ctrl;
int ret;
ret = nvme_tcp_alloc_queue(ctrl, ctx->qid, ctrl->tls_pskid);
if (ret)
goto out_err;
ret = nvme_tcp_start_queue(ctrl, ctx->qid);
if (ret)
goto out_err;
return;
out_err:
WRITE_ONCE(*ctx->err, ret);
}
static int nvme_tcp_setup_io_queues(struct nvme_ctrl *ctrl, unsigned int first,
unsigned int last)
{
ASYNC_DOMAIN_EXCLUSIVE(queue_domain);
struct nvme_tcp_queue_setup_ctx *ctxs;
int nr_queues = last - first;
int err = 0, i, ret;
if (first >= last)
return 0;
ctxs = kmalloc_objs(*ctxs, nr_queues);
if (!ctxs)
return -ENOMEM;
for (i = 0; i < nr_queues; i++) {
ctxs[i].ctrl = ctrl;
ctxs[i].qid = first + i;
ctxs[i].err = &err;
async_schedule_domain(nvme_tcp_setup_queue_async, &ctxs[i],
&queue_domain);
}
async_synchronize_full_domain(&queue_domain);
kfree(ctxs);
ret = READ_ONCE(err);
if (ret)
goto out_free_queues;
return 0;
out_free_queues:
for (i--; i >= 1; i--)
nvme_tcp_free_queue(ctrl, i);
for (i = first; i < last; i++) {
struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[i];
if (test_bit(NVME_TCP_Q_LIVE, &queue->flags))
nvme_tcp_stop_queue(ctrl, i);
if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
nvme_tcp_free_queue(ctrl, i);
}
return ret;
}
static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
static int nvme_tcp_set_io_queue_count(struct nvme_ctrl *ctrl)
{
unsigned int nr_io_queues;
int ret;
@@ -2233,14 +2278,18 @@ static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
nvmf_set_io_queues(ctrl->opts, nr_io_queues,
to_tcp_ctrl(ctrl)->io_queues);
return __nvme_tcp_alloc_io_queues(ctrl);
return 0;
}
static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
{
int ret, nr_queues;
ret = nvme_tcp_alloc_io_queues(ctrl);
ret = nvme_tcp_set_io_queue_count(ctrl);
if (ret)
return ret;
ret = nvme_tcp_tls_check_psk(ctrl);
if (ret)
return ret;
@@ -2254,12 +2303,12 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
}
/*
* Only start IO queues for which we have allocated the tagset
* Only setup IO queues for which we have allocated the tagset
* and limited it to the available queues. On reconnects, the
* queue number might have changed.
*/
nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues);
ret = nvme_tcp_setup_io_queues(ctrl, 1, nr_queues);
if (ret)
goto out_cleanup_connect_q;
@@ -2283,9 +2332,9 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
/*
* If the number of queues has increased (reconnect case)
* start all new queues now.
* setup all new queues now.
*/
ret = nvme_tcp_start_io_queues(ctrl, nr_queues,
ret = nvme_tcp_setup_io_queues(ctrl, nr_queues,
ctrl->tagset->nr_hw_queues + 1);
if (ret)
goto out_wait_freeze_timed_out;
@@ -2637,6 +2686,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
nvmf_free_options(nctrl->opts);
free_ctrl:
put_net(ctrl->net);
kfree(ctrl->queues);
kfree(ctrl);
}
@@ -3035,12 +3085,15 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
goto out_free_ctrl;
}
ctrl->net = get_net(current->nsproxy->net_ns);
ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
if (ret)
goto out_kfree_queues;
return ctrl;
out_kfree_queues:
put_net(ctrl->net);
kfree(ctrl->queues);
out_free_ctrl:
kfree(ctrl);
@@ -3126,7 +3179,7 @@ static int __init nvme_tcp_init_module(void)
return -ENOMEM;
for_each_possible_cpu(cpu)
atomic_set(&nvme_tcp_cpu_queues[cpu], 0);
nvme_tcp_cpu_queues[cpu] = 0;
nvmf_register_transport(&nvme_tcp_transport);
return 0;
drivers/nvme/target/admin-cmd.c
+3 -1
View File
@@ -696,7 +696,9 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
/* XXX: figure out what to do about RTD3R/RTD3 */
id->oaes = cpu_to_le32(NVMET_AEN_CFG_OPTIONAL);
ctratt = NVME_CTRL_ATTR_HID_128_BIT | NVME_CTRL_ATTR_TBKAS;
ctratt = NVME_CTRL_ATTR_HID_128_BIT;
if (ctrl->tbkas)
ctratt |= NVME_CTRL_ATTR_TBKAS;
if (nvmet_is_pci_ctrl(ctrl))
ctratt |= NVME_CTRL_ATTR_RHII;
id->ctratt = cpu_to_le32(ctratt);
drivers/nvme/target/auth.c
+1
View File
@@ -229,6 +229,7 @@ out_unlock:
void nvmet_auth_sq_free(struct nvmet_sq *sq)
{
sq->auth_expiry_active = false;
cancel_delayed_work(&sq->auth_expired_work);
kfree(sq->dhchap_c1);
sq->dhchap_c1 = NULL;
drivers/nvme/target/configfs.c
+107 -12
View File
@@ -905,21 +905,34 @@ static ssize_t nvmet_passthru_device_path_show(struct config_item *item,
char *page)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
ssize_t ret;
return snprintf(page, PAGE_SIZE, "%s\n", subsys->passthru_ctrl_path);
mutex_lock(&subsys->lock);
ret = snprintf(page, PAGE_SIZE, "%s\n",
subsys->passthru && subsys->passthru->ctrl_path ?
subsys->passthru->ctrl_path : "");
mutex_unlock(&subsys->lock);
return ret;
}
static ssize_t nvmet_passthru_device_path_store(struct config_item *item,
const char *page, size_t count)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
struct nvmet_passthru *passthru;
size_t len;
int ret;
mutex_lock(&subsys->lock);
passthru = nvmet_subsys_passthru(subsys);
if (!passthru) {
ret = -ENOMEM;
goto out_unlock;
}
ret = -EBUSY;
if (subsys->passthru_ctrl)
if (passthru->ctrl)
goto out_unlock;
ret = -EINVAL;
@@ -927,10 +940,10 @@ static ssize_t nvmet_passthru_device_path_store(struct config_item *item,
if (!len)
goto out_unlock;
kfree(subsys->passthru_ctrl_path);
kfree(passthru->ctrl_path);
ret = -ENOMEM;
subsys->passthru_ctrl_path = kstrndup(page, len, GFP_KERNEL);
if (!subsys->passthru_ctrl_path)
passthru->ctrl_path = kstrndup(page, len, GFP_KERNEL);
if (!passthru->ctrl_path)
goto out_unlock;
mutex_unlock(&subsys->lock);
@@ -946,8 +959,13 @@ static ssize_t nvmet_passthru_enable_show(struct config_item *item,
char *page)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
ssize_t ret;
return sprintf(page, "%d\n", subsys->passthru_ctrl ? 1 : 0);
mutex_lock(&subsys->lock);
ret = sprintf(page, "%d\n",
subsys->passthru && subsys->passthru->ctrl ? 1 : 0);
mutex_unlock(&subsys->lock);
return ret;
}
static ssize_t nvmet_passthru_enable_store(struct config_item *item,
@@ -972,18 +990,34 @@ CONFIGFS_ATTR(nvmet_passthru_, enable);
static ssize_t nvmet_passthru_admin_timeout_show(struct config_item *item,
char *page)
{
return sprintf(page, "%u\n", to_subsys(item->ci_parent)->admin_timeout);
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
ssize_t ret;
mutex_lock(&subsys->lock);
ret = sprintf(page, "%u\n", subsys->passthru ?
subsys->passthru->admin_timeout : 0);
mutex_unlock(&subsys->lock);
return ret;
}
static ssize_t nvmet_passthru_admin_timeout_store(struct config_item *item,
const char *page, size_t count)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
struct nvmet_passthru *passthru;
unsigned int timeout;
if (kstrtouint(page, 0, &timeout))
return -EINVAL;
subsys->admin_timeout = timeout;
mutex_lock(&subsys->lock);
passthru = nvmet_subsys_passthru(subsys);
if (!passthru) {
mutex_unlock(&subsys->lock);
return -ENOMEM;
}
passthru->admin_timeout = timeout;
mutex_unlock(&subsys->lock);
return count;
}
CONFIGFS_ATTR(nvmet_passthru_, admin_timeout);
@@ -991,18 +1025,34 @@ CONFIGFS_ATTR(nvmet_passthru_, admin_timeout);
static ssize_t nvmet_passthru_io_timeout_show(struct config_item *item,
char *page)
{
return sprintf(page, "%u\n", to_subsys(item->ci_parent)->io_timeout);
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
ssize_t ret;
mutex_lock(&subsys->lock);
ret = sprintf(page, "%u\n",
subsys->passthru ? subsys->passthru->io_timeout : 0);
mutex_unlock(&subsys->lock);
return ret;
}
static ssize_t nvmet_passthru_io_timeout_store(struct config_item *item,
const char *page, size_t count)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
struct nvmet_passthru *passthru;
unsigned int timeout;
if (kstrtouint(page, 0, &timeout))
return -EINVAL;
subsys->io_timeout = timeout;
mutex_lock(&subsys->lock);
passthru = nvmet_subsys_passthru(subsys);
if (!passthru) {
mutex_unlock(&subsys->lock);
return -ENOMEM;
}
passthru->io_timeout = timeout;
mutex_unlock(&subsys->lock);
return count;
}
CONFIGFS_ATTR(nvmet_passthru_, io_timeout);
@@ -1010,18 +1060,34 @@ CONFIGFS_ATTR(nvmet_passthru_, io_timeout);
static ssize_t nvmet_passthru_clear_ids_show(struct config_item *item,
char *page)
{
return sprintf(page, "%u\n", to_subsys(item->ci_parent)->clear_ids);
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
ssize_t ret;
mutex_lock(&subsys->lock);
ret = sprintf(page, "%u\n",
subsys->passthru ? subsys->passthru->clear_ids : 0);
mutex_unlock(&subsys->lock);
return ret;
}
static ssize_t nvmet_passthru_clear_ids_store(struct config_item *item,
const char *page, size_t count)
{
struct nvmet_subsys *subsys = to_subsys(item->ci_parent);
struct nvmet_passthru *passthru;
unsigned int clear_ids;
if (kstrtouint(page, 0, &clear_ids))
return -EINVAL;
subsys->clear_ids = clear_ids;
mutex_lock(&subsys->lock);
passthru = nvmet_subsys_passthru(subsys);
if (!passthru) {
mutex_unlock(&subsys->lock);
return -ENOMEM;
}
passthru->clear_ids = clear_ids;
mutex_unlock(&subsys->lock);
return count;
}
CONFIGFS_ATTR(nvmet_passthru_, clear_ids);
@@ -1372,6 +1438,34 @@ static ssize_t nvmet_subsys_attr_serial_store(struct config_item *item,
}
CONFIGFS_ATTR(nvmet_subsys_, attr_serial);
static ssize_t nvmet_subsys_attr_tbkas_show(struct config_item *item,
char *page)
{
return snprintf(page, PAGE_SIZE, "%d\n", to_subsys(item)->tbkas);
}
static ssize_t nvmet_subsys_attr_tbkas_store(struct config_item *item,
const char *page, size_t count)
{
struct nvmet_subsys *subsys = to_subsys(item);
struct nvmet_ctrl *ctrl;
bool tbkas;
if (kstrtobool(page, &tbkas))
return -EINVAL;
down_write(&nvmet_config_sem);
subsys->tbkas = tbkas;
/* Force reconnect */
list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
ctrl->ops->delete_ctrl(ctrl);
up_write(&nvmet_config_sem);
return count;
}
CONFIGFS_ATTR(nvmet_subsys_, attr_tbkas);
static ssize_t nvmet_subsys_attr_cntlid_min_show(struct config_item *item,
char *page)
{
@@ -1709,6 +1803,7 @@ static struct configfs_attribute *nvmet_subsys_attrs[] = {
&nvmet_subsys_attr_attr_qid_max,
&nvmet_subsys_attr_attr_ieee_oui,
&nvmet_subsys_attr_attr_firmware,
&nvmet_subsys_attr_attr_tbkas,
#ifdef CONFIG_BLK_DEV_INTEGRITY
&nvmet_subsys_attr_attr_pi_enable,
#endif
drivers/nvme/target/core.c
+16 -3
View File
@@ -819,6 +819,8 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
nvmet_pr_put_ns_pc_ref(pc_ref);
if (ns)
nvmet_put_namespace(ns);
nvmet_put_passthru_ref(req);
}
void nvmet_req_complete(struct nvmet_req *req, u16 status)
@@ -1195,6 +1197,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_sq *sq,
req->error_loc = NVMET_NO_ERROR_LOC;
req->error_slba = 0;
req->pc_ref = NULL;
req->p.ref_held = false;
/* no support for fused commands yet */
if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) {
@@ -1235,7 +1238,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_sq *sq,
goto fail;
}
if (sq->ctrl)
if (sq->ctrl && sq->ctrl->tbkas)
sq->ctrl->reset_tbkas = true;
return true;
@@ -1645,8 +1648,16 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
#ifdef CONFIG_NVME_TARGET_PASSTHRU
/* By default, set loop targets to clear IDS by default */
if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP)
subsys->clear_ids = 1;
if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP) {
mutex_lock(&subsys->lock);
subsys->passthru = nvmet_subsys_passthru(subsys);
if (!subsys->passthru) {
mutex_unlock(&subsys->lock);
goto out_free_ctrl;
}
subsys->passthru->clear_ids = 1;
mutex_unlock(&subsys->lock);
}
#endif
INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);
@@ -1687,6 +1698,7 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
mutex_lock(&subsys->lock);
ctrl->max_qid = subsys->max_qid;
ctrl->tbkas = subsys->tbkas;
ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1);
if (!ctrl->sqs)
@@ -1872,6 +1884,7 @@ struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn,
}
subsys->ieee_oui = 0;
subsys->tbkas = true;
subsys->firmware_rev = kstrndup(UTS_RELEASE, NVMET_FR_MAX_SIZE, GFP_KERNEL);
if (!subsys->firmware_rev) {
drivers/nvme/target/fabrics-cmd-auth.c
+34 -4
View File
@@ -6,6 +6,7 @@
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/blkdev.h>
#include <linux/jiffies.h>
#include <linux/random.h>
#include <linux/nvme-auth.h>
#include <crypto/kpp.h>
@@ -17,16 +18,27 @@ static void nvmet_auth_expired_work(struct work_struct *work)
struct nvmet_sq *sq = container_of(to_delayed_work(work),
struct nvmet_sq, auth_expired_work);
mutex_lock(&sq->auth_lock);
/* A command may have cancelled or rearmed this timeout. */
if (!sq->auth_expiry_active ||
delayed_work_pending(&sq->auth_expired_work))
goto out_unlock;
sq->auth_expiry_active = false;
pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
__func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
sq->dhchap_tid = -1;
out_unlock:
mutex_unlock(&sq->auth_lock);
}
void nvmet_auth_sq_init(struct nvmet_sq *sq)
{
/* Initialize in-band authentication */
mutex_init(&sq->auth_lock);
INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work);
sq->auth_expiry_active = false;
sq->authenticated = false;
sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
}
@@ -256,6 +268,8 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
u16 status = 0;
u8 dhchap_status;
mutex_lock(&req->sq->auth_lock);
if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
req->error_loc =
@@ -281,10 +295,18 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
offsetof(struct nvmf_auth_send_command, tl);
goto done;
}
if (!nvmet_check_transfer_len(req, tl)) {
if (unlikely(tl != req->transfer_len)) {
pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
mutex_unlock(&req->sq->auth_lock);
nvmet_check_transfer_len(req, tl);
return;
}
if (tl < offsetof(struct nvmf_auth_dhchap_success2_data, rsvd1)) {
status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
req->error_loc =
offsetof(struct nvmf_auth_send_command, tl);
goto done;
}
d = kmalloc(tl, GFP_KERNEL);
if (!d) {
@@ -406,10 +428,12 @@ done:
status, req->error_loc);
if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
unsigned long auth_expire = min_t(u64, MAX_JIFFY_OFFSET,
(u64)(ctrl->kato ? ctrl->kato : 120) * HZ);
req->sq->auth_expiry_active = true;
mod_delayed_work(system_percpu_wq, &req->sq->auth_expired_work,
auth_expire_secs * HZ);
auth_expire);
goto complete;
}
/* Final states, clear up variables */
@@ -418,6 +442,7 @@ done:
nvmet_ctrl_fatal_error(ctrl);
complete:
mutex_unlock(&req->sq->auth_lock);
nvmet_req_complete(req, status);
}
@@ -536,6 +561,8 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
u32 al;
u16 status = 0;
mutex_lock(&req->sq->auth_lock);
if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
req->error_loc =
@@ -561,8 +588,10 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
offsetof(struct nvmf_auth_receive_command, al);
goto done;
}
if (!nvmet_check_transfer_len(req, al)) {
if (unlikely(al != req->transfer_len)) {
pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
mutex_unlock(&req->sq->auth_lock);
nvmet_check_transfer_len(req, al);
return;
}
@@ -621,5 +650,6 @@ done:
nvmet_auth_sq_free(req->sq);
nvmet_ctrl_fatal_error(ctrl);
}
mutex_unlock(&req->sq->auth_lock);
nvmet_req_complete(req, status);
}
drivers/nvme/target/fabrics-cmd.c
+4
View File
@@ -14,6 +14,10 @@ static void nvmet_execute_prop_set(struct nvmet_req *req)
if (!nvmet_check_transfer_len(req, 0))
return;
if (!nvmet_check_auth_status(req)) {
status = NVME_SC_AUTH_REQUIRED | NVME_STATUS_DNR;
goto out;
}
if (req->cmd->prop_set.attrib & 1) {
req->error_loc =
drivers/nvme/target/fc.c
+9 -6
View File
@@ -1567,8 +1567,14 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
queue = assoc->queues[0];
if (queue && queue->nvme_sq.ctrl == ctrl) {
if (nvmet_fc_tgt_a_get(assoc))
found_ctrl = true;
/*
* No assoc reference here: we're called with
* subsys->lock held, and if del_work beat us,
* our put would free the ctrl under that lock.
* RCU keeps the assoc alive until unlock.
*/
nvmet_fc_schedule_delete_assoc(assoc);
found_ctrl = true;
break;
}
}
@@ -1576,11 +1582,8 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
nvmet_fc_tgtport_put(tgtport);
if (found_ctrl) {
nvmet_fc_schedule_delete_assoc(assoc);
nvmet_fc_tgt_a_put(assoc);
if (found_ctrl)
return;
}
spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
}
drivers/nvme/target/io-cmd-file.c
+2 -2
View File
@@ -133,8 +133,8 @@ static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
if (unlikely(pos + req->transfer_len > req->ns->size)) {
nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
return true;
ret = -ENOSPC;
goto complete;
}
memset(&req->f.iocb, 0, sizeof(struct kiocb));
drivers/nvme/target/nvmet.h
+64 -6
View File
@@ -161,8 +161,10 @@ struct nvmet_sq {
u32 sqhd;
bool sqhd_disabled;
#ifdef CONFIG_NVME_TARGET_AUTH
struct mutex auth_lock;
bool authenticated;
struct delayed_work auth_expired_work;
bool auth_expiry_active;
u16 dhchap_tid;
u8 sc_c;
u8 dhchap_status;
@@ -272,6 +274,7 @@ struct nvmet_ctrl {
u16 cntlid;
u16 max_qid;
u32 kato;
bool tbkas;
struct nvmet_port *port;
@@ -319,6 +322,19 @@ struct nvmet_ctrl {
struct nvmet_pr_log_mgr pr_log_mgr;
};
struct nvmet_passthru {
struct percpu_ref ref;
struct completion disable_done;
#define NVMET_PASSTHRU_ENABLED 0
unsigned long flags;
struct nvme_ctrl *ctrl;
char *ctrl_path;
unsigned int admin_timeout;
unsigned int io_timeout;
unsigned int clear_ids;
};
struct nvmet_subsys {
enum nvme_subsys_type type;
@@ -345,6 +361,7 @@ struct nvmet_subsys {
bool subsys_discovered;
char *subsysnqn;
bool pi_support;
bool tbkas;
struct config_group group;
@@ -358,12 +375,8 @@ struct nvmet_subsys {
char *firmware_rev;
#ifdef CONFIG_NVME_TARGET_PASSTHRU
struct nvme_ctrl *passthru_ctrl;
char *passthru_ctrl_path;
struct nvmet_passthru *passthru;
struct config_group passthru_group;
unsigned int admin_timeout;
unsigned int io_timeout;
unsigned int clear_ids;
#endif /* CONFIG_NVME_TARGET_PASSTHRU */
#ifdef CONFIG_BLK_DEV_ZONED
@@ -474,6 +487,7 @@ struct nvmet_req {
struct request *rq;
struct work_struct work;
bool use_workqueue;
bool ref_held;
} p;
#ifdef CONFIG_BLK_DEV_ZONED
struct {
@@ -793,7 +807,39 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req);
u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req);
static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys)
{
return subsys->passthru_ctrl;
return subsys->passthru &&
test_bit(NVMET_PASSTHRU_ENABLED, &subsys->passthru->flags);
}
static inline struct nvmet_passthru *nvmet_subsys_passthru(
struct nvmet_subsys *subsys)
{
lockdep_assert_held(&subsys->lock);
if (!subsys->passthru) {
subsys->passthru = kzalloc_obj(*subsys->passthru);
if (!subsys->passthru)
return NULL;
}
return subsys->passthru;
}
static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
{
if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref))
return false;
req->p.ref_held = true;
return true;
}
static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
{
if (!req->p.ref_held)
return;
req->p.ref_held = false;
percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref);
}
#else /* CONFIG_NVME_TARGET_PASSTHRU */
static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
@@ -814,6 +860,18 @@ static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys)
{
return NULL;
}
static inline struct nvmet_passthru *nvmet_subsys_passthru(
struct nvmet_subsys *subsys)
{
return NULL;
}
static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
{
return false;
}
static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
{
}
#endif /* CONFIG_NVME_TARGET_PASSTHRU */
static inline bool nvmet_is_passthru_req(struct nvmet_req *req)
drivers/nvme/target/passthru.c
+81 -22
View File
@@ -26,7 +26,7 @@ void nvmet_passthrough_override_cap(struct nvmet_ctrl *ctrl)
* Multiple command set support can only be declared if the underlying
* controller actually supports it.
*/
if (!nvme_multi_css(ctrl->subsys->passthru_ctrl))
if (!nvme_multi_css(ctrl->subsys->passthru->ctrl))
ctrl->cap &= ~(1ULL << 43);
}
@@ -39,7 +39,7 @@ static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req)
void *data;
u8 csi;
if (!ctrl->subsys->clear_ids)
if (!ctrl->subsys->passthru->clear_ids)
return status;
data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
@@ -89,7 +89,7 @@ out_free:
static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req)
{
struct nvmet_ctrl *ctrl = req->sq->ctrl;
struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl;
struct nvme_ctrl *pctrl = ctrl->subsys->passthru->ctrl;
u16 status = NVME_SC_SUCCESS;
struct nvme_id_ctrl *id;
unsigned int max_hw_sectors;
@@ -208,7 +208,7 @@ static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req)
*/
id->mc = 0;
if (req->sq->ctrl->subsys->clear_ids) {
if (req->sq->ctrl->subsys->passthru->clear_ids) {
memset(id->nguid, 0, NVME_NIDT_NGUID_LEN);
memset(id->eui64, 0, NVME_NIDT_EUI64_LEN);
}
@@ -305,8 +305,9 @@ out_bio_put:
static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
{
struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
struct request_queue *q = ctrl->admin_q;
struct nvmet_passthru *passthru;
struct nvme_ctrl *ctrl;
struct request_queue *q;
struct nvme_ns *ns = NULL;
struct request *rq = NULL;
unsigned int timeout;
@@ -314,6 +315,15 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
u16 status;
int ret;
if (!nvmet_get_passthru_ref(req)) {
status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
goto out;
}
passthru = nvmet_req_subsys(req)->passthru;
ctrl = passthru->ctrl;
q = ctrl->admin_q;
if (likely(req->sq->qid != 0)) {
u32 nsid = le32_to_cpu(req->cmd->common.nsid);
@@ -325,9 +335,9 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
}
q = ns->queue;
timeout = nvmet_req_subsys(req)->io_timeout;
timeout = passthru->io_timeout;
} else {
timeout = nvmet_req_subsys(req)->admin_timeout;
timeout = passthru->admin_timeout;
}
rq = blk_mq_alloc_request(q, nvme_req_op(req->cmd), 0);
@@ -386,11 +396,17 @@ out:
*/
static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req)
{
struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
struct nvme_ctrl *ctrl;
struct nvme_feat_host_behavior *host;
u16 status = NVME_SC_INTERNAL;
int ret;
if (!nvmet_get_passthru_ref(req)) {
status |= NVME_STATUS_DNR;
goto out_complete_req;
}
ctrl = nvmet_req_subsys(req)->passthru->ctrl;
host = kzalloc(sizeof(*host) * 2, GFP_KERNEL);
if (!host)
goto out_complete_req;
@@ -584,17 +600,27 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
}
}
static void nvmet_release_passthru_ctrl(struct percpu_ref *ref)
{
struct nvmet_passthru *passthru = container_of(ref,
struct nvmet_passthru, ref);
complete(&passthru->disable_done);
}
int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
{
struct nvmet_passthru *passthru;
struct nvme_ctrl *ctrl;
struct file *file;
int ret = -EINVAL;
void *old;
mutex_lock(&subsys->lock);
if (!subsys->passthru_ctrl_path)
passthru = subsys->passthru;
if (!passthru || !passthru->ctrl_path)
goto out_unlock;
if (subsys->passthru_ctrl)
if (passthru->ctrl)
goto out_unlock;
if (subsys->nr_namespaces) {
@@ -602,7 +628,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
goto out_unlock;
}
file = filp_open(subsys->passthru_ctrl_path, O_RDWR, 0);
file = filp_open(passthru->ctrl_path, O_RDWR, 0);
if (IS_ERR(file)) {
ret = PTR_ERR(file);
goto out_unlock;
@@ -611,7 +637,7 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
ctrl = nvme_ctrl_from_file(file);
if (!ctrl) {
pr_err("failed to open nvme controller %s\n",
subsys->passthru_ctrl_path);
passthru->ctrl_path);
goto out_put_file;
}
@@ -626,7 +652,14 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
if (old)
goto out_put_file;
subsys->passthru_ctrl = ctrl;
ret = percpu_ref_init(&passthru->ref, nvmet_release_passthru_ctrl,
0, GFP_KERNEL);
if (ret) {
xa_erase(&passthru_subsystems, ctrl->instance);
goto out_put_file;
}
init_completion(&passthru->disable_done);
passthru->ctrl = ctrl;
subsys->ver = ctrl->vs;
if (subsys->ver < NVME_VS(1, 2, 1)) {
@@ -636,7 +669,8 @@ int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
subsys->ver = NVME_VS(1, 2, 1);
}
nvme_get_ctrl(ctrl);
__module_get(subsys->passthru_ctrl->ops->module);
__module_get(passthru->ctrl->ops->module);
set_bit(NVMET_PASSTHRU_ENABLED, &passthru->flags);
ret = 0;
out_put_file:
@@ -648,12 +682,33 @@ out_unlock:
static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
{
if (subsys->passthru_ctrl) {
xa_erase(&passthru_subsystems, subsys->passthru_ctrl->instance);
module_put(subsys->passthru_ctrl->ops->module);
nvme_put_ctrl(subsys->passthru_ctrl);
}
subsys->passthru_ctrl = NULL;
struct nvmet_passthru *passthru = subsys->passthru;
lockdep_assert_held(&subsys->lock);
if (!passthru)
return;
if (!test_and_clear_bit(NVMET_PASSTHRU_ENABLED, &passthru->flags))
return;
mutex_unlock(&subsys->lock);
/*
* Now new I/Os should not enter passthru hotpath as we cleared the
* enabled flag. Kill percpu reference and wait for in-flight I/Os
* to drain.
*/
percpu_ref_kill(&passthru->ref);
wait_for_completion(&passthru->disable_done);
percpu_ref_exit(&passthru->ref);
mutex_lock(&subsys->lock);
xa_erase(&passthru_subsystems, passthru->ctrl->instance);
module_put(passthru->ctrl->ops->module);
nvme_put_ctrl(passthru->ctrl);
passthru->ctrl = NULL;
subsys->ver = NVMET_DEFAULT_VS;
}
@@ -668,6 +723,10 @@ void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
{
mutex_lock(&subsys->lock);
__nvmet_passthru_ctrl_disable(subsys);
if (subsys->passthru) {
kfree(subsys->passthru->ctrl_path);
kfree(subsys->passthru);
subsys->passthru = NULL;
}
mutex_unlock(&subsys->lock);
kfree(subsys->passthru_ctrl_path);
}
drivers/nvme/target/pr.c
+2 -2
View File
@@ -589,11 +589,11 @@ static u16 nvmet_pr_preempt(struct nvmet_req *req,
&ctrl->hostid, abort);
}
if (holder == reg) {
if (holder == reg && prkey == holder->rkey) {
status = nvmet_pr_update_reg_attr(pr, holder,
nvmet_pr_update_holder_rtype, &rtype);
if (!status && original_rtype != rtype)
nvmet_pr_resv_released(pr, &reg->hostid);
nvmet_pr_resv_released(pr, &ctrl->hostid);
return status;
}
drivers/nvme/target/zns.c
+10
View File
@@ -282,6 +282,11 @@ static void nvmet_bdev_zone_zmgmt_recv_work(struct work_struct *w)
if (status)
goto out;
/* Only the zone count and matching descriptors are written below. */
status = nvmet_zero_sgl(req, 0, out_bufsize);
if (status)
goto out;
if (!req_slba_nr_zones) {
status = NVME_SC_SUCCESS;
goto out;
@@ -317,6 +322,11 @@ out:
void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req)
{
u32 out_bufsize = (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
if (!nvmet_check_transfer_len(req, out_bufsize))
return;
INIT_WORK(&req->z.zmgmt_work, nvmet_bdev_zone_zmgmt_recv_work);
queue_work(zbd_wq, &req->z.zmgmt_work);
}
fs/iomap/buffered-io.c
+4
View File
@@ -1304,6 +1304,10 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
};
ssize_t ret;
/* Buffered atomic writes are not supported */
if (iocb->ki_flags & IOCB_ATOMIC)
return -EOPNOTSUPP;
if (iocb->ki_flags & IOCB_NOWAIT)
iter.flags |= IOMAP_NOWAIT;
if (iocb->ki_flags & IOCB_DONTCACHE)
include/linux/blk-mq.h
-3
View File
@@ -525,8 +525,6 @@ enum hctx_type {
* request_queue.tag_set_list.
* @srcu: Use as lock when type of the request queue is blocking
* (BLK_MQ_F_BLOCKING).
* @tags_srcu: SRCU used to defer freeing of tags page_list to prevent
* use-after-free when iterating tags.
* @update_nr_hwq_lock:
* Synchronize updating nr_hw_queues with add/del disk &
* switching elevator.
@@ -551,7 +549,6 @@ struct blk_mq_tag_set {
struct mutex tag_list_lock;
struct list_head tag_list;
struct srcu_struct *srcu;
struct srcu_struct tags_srcu;
struct rw_semaphore update_nr_hwq_lock;
};
include/linux/blkdev.h
+7 -4
View File
@@ -191,14 +191,17 @@ struct gendisk {
#ifdef CONFIG_BLK_DEV_ZONED
/*
* Zoned block device information. Reads of this information must be
* protected with blk_queue_enter() / blk_queue_exit(). Modifying this
* information is only allowed while no requests are being processed.
* See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
* protected with blk_queue_enter() / blk_queue_exit() or by holding a
* lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify
* this information while no requests are being processed (disk queue
* frozen with blk_mq_freeze_queue()) and while holding a lock on
* zone_revalidate_mutex.
*/
struct mutex zone_revalidate_mutex;
unsigned int nr_zones;
unsigned int zone_capacity;
unsigned int last_zone_capacity;
u8 __rcu *zones_cond;
u8 __rcu *zones_state;
unsigned int zone_wplugs_hash_bits;
atomic_t nr_zone_wplugs;
spinlock_t zone_wplugs_hash_lock;
rust/kernel/block/mq/gen_disk.rs
+38 -34
View File
@@ -7,14 +7,26 @@
use crate::{
bindings,
block::mq::{Operations, TagSet},
error::{self, from_err_ptr, Result},
fmt::{self, Write},
block::mq::{
Operations,
TagSet, //
},
error::{
from_err_ptr,
to_result, //
},
fmt::{
self,
Write, //
},
prelude::*,
static_lock_class,
str::NullTerminatedFormatter,
sync::Arc,
types::{ForeignOwnable, ScopeGuard},
types::{
ForeignOwnable,
ScopeGuard, //
}, //
};
/// A builder for [`GenDisk`].
@@ -54,7 +66,7 @@ impl GenDiskBuilder {
/// and that it is a power of two.
pub fn validate_block_size(size: u32) -> Result {
if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() {
Err(error::code::EINVAL)
Err(EINVAL)
} else {
Ok(())
}
@@ -63,7 +75,7 @@ impl GenDiskBuilder {
/// Set the logical block size of the device to be built.
///
/// This method will check that block size is a power of two and between 512
/// and 4096. If not, an error is returned and the block size is not set.
/// and `PAGE_SIZE`. If not, an error is returned and the block size is not set.
///
/// This is the smallest unit the storage device can address. It is
/// typically 4096 bytes.
@@ -76,7 +88,7 @@ impl GenDiskBuilder {
/// Set the physical block size of the device to be built.
///
/// This method will check that block size is a power of two and between 512
/// and 4096. If not, an error is returned and the block size is not set.
/// and `PAGE_SIZE`. If not, an error is returned and the block size is not set.
///
/// This is the smallest unit a physical storage device can write
/// atomically. It is usually the same as the logical block size but may be
@@ -125,30 +137,9 @@ impl GenDiskBuilder {
)
})?;
const TABLE: bindings::block_device_operations = bindings::block_device_operations {
submit_bio: None,
open: None,
release: None,
ioctl: None,
compat_ioctl: None,
check_events: None,
unlock_native_capacity: None,
getgeo: None,
set_read_only: None,
swap_slot_free_notify: None,
report_zones: None,
devnode: None,
alternative_gpt_sector: None,
get_unique_id: None,
// TODO: Set to `THIS_MODULE`.
owner: core::ptr::null_mut(),
pr_ops: core::ptr::null_mut(),
free_disk: None,
poll_bio: None,
};
// SAFETY: `gendisk` is a valid pointer as we initialized it above
unsafe { (*gendisk).fops = &TABLE };
// SAFETY: `gendisk` is a valid pointer. We have exclusive access,
// since the disk is not added to the VFS yet.
unsafe { (*gendisk).fops = &GenDisk::<T>::VTABLE };
let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| {
// SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and
@@ -177,7 +168,7 @@ impl GenDiskBuilder {
// operation, so we will not race.
unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) };
crate::error::to_result(
to_result(
// SAFETY: `gendisk` points to a valid and initialized instance of
// `struct gendisk`.
unsafe {
@@ -211,9 +202,22 @@ pub struct GenDisk<T: Operations> {
gendisk: *mut bindings::gendisk,
}
impl<T: Operations> GenDisk<T> {
const VTABLE: bindings::block_device_operations = bindings::block_device_operations {
owner: crate::module::this_module::<T::OwnerModule>().as_ptr(),
..pin_init::zeroed()
};
}
// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a
// `TagSet` It is safe to send this to other threads as long as T is Send.
unsafe impl<T: Operations + Send> Send for GenDisk<T> {}
// `TagSet`. It is safe to send this to other threads as long as these two are `Send`.
unsafe impl<T> Send for GenDisk<T>
where
T: Operations,
T::QueueData: Send,
Arc<TagSet<T>>: Send,
{
}
impl<T: Operations> Drop for GenDisk<T> {
fn drop(&mut self) {
rust/kernel/block/mq/operations.rs
+11 -5
View File
@@ -6,11 +6,17 @@
use crate::{
bindings,
block::mq::{request::RequestDataWrapper, Request},
error::{from_result, Result},
block::mq::{
request::RequestDataWrapper,
Request, //
},
error::from_result,
prelude::*,
sync::{aref::ARef, Refcount},
types::ForeignOwnable,
sync::{
aref::ARef,
Refcount, //
},
types::ForeignOwnable, //
};
use core::marker::PhantomData;
@@ -30,7 +36,7 @@ type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>;
pub trait Operations: Sized {
/// Data associated with the `struct request_queue` that is allocated for
/// the `GenDisk` associated with this `Operations` implementation.
type QueueData: ForeignOwnable;
type QueueData: ForeignOwnable + Sync;
/// Called by the kernel to queue a request with the driver. If `is_last` is
/// `false`, the driver is allowed to defer committing the request.
rust/kernel/block/mq/request.rs
+11 -5
View File
@@ -7,15 +7,21 @@
use crate::{
bindings,
block::mq::Operations,
error::Result,
prelude::*,
sync::{
aref::{ARef, AlwaysRefCounted},
aref::{
ARef,
AlwaysRefCounted, //
},
atomic::Relaxed,
Refcount,
Refcount, //
},
types::Opaque,
types::Opaque, //
};
use core::{
marker::PhantomData,
ptr::NonNull, //
};
use core::{marker::PhantomData, ptr::NonNull};
/// A wrapper around a blk-mq [`struct request`]. This represents an IO request.
///
rust/kernel/block/mq/tag_set.rs
+21 -14
View File
@@ -4,17 +4,21 @@
//!
//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
use core::pin::Pin;
use crate::{
bindings,
block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations},
error::{self, Result},
prelude::try_pin_init,
types::Opaque,
block::mq::{
operations::OperationsVTable,
request::RequestDataWrapper,
Operations, //
},
error::to_result,
prelude::*,
types::Opaque, //
};
use core::{
convert::TryInto,
marker::PhantomData, //
};
use core::{convert::TryInto, marker::PhantomData};
use pin_init::{pin_data, pinned_drop, PinInit};
/// A wrapper for the C `struct blk_mq_tag_set`.
///
@@ -33,11 +37,7 @@ pub struct TagSet<T: Operations> {
impl<T: Operations> TagSet<T> {
/// Try to create a new tag set
pub fn new(
nr_hw_queues: u32,
num_tags: u32,
num_maps: u32,
) -> impl PinInit<Self, error::Error> {
pub fn new(nr_hw_queues: u32, num_tags: u32, num_maps: u32) -> impl PinInit<Self, Error> {
let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed();
let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>()
.try_into()
@@ -63,7 +63,7 @@ impl<T: Operations> TagSet<T> {
// SAFETY: we do not move out of `tag_set`.
let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) };
// SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`.
error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())})
to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())})
}),
_p: PhantomData,
})
@@ -83,3 +83,10 @@ impl<T: Operations> PinnedDrop for TagSet<T> {
unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) };
}
}
// SAFETY: It is safe to transfer ownership of `TagSet` between threads.
unsafe impl<T: Operations> Send for TagSet<T> {}
// SAFETY: It is safe to share `&TagSet` across threads. `TagSet` exposes no
// `&self` methods that mutate the contained `blk_mq_tag_set`.
unsafe impl<T: Operations> Sync for TagSet<T> {}
tools/testing/selftests/ublk/.gitignore
+1
View File
@@ -3,3 +3,4 @@
/tools
kublk
metadata_size
ublk_cancel_ready
tools/testing/selftests/ublk/Makefile
+4 -2
View File
@@ -19,6 +19,7 @@ TEST_PROGS += test_generic_12.sh
TEST_PROGS += test_generic_13.sh
TEST_PROGS += test_generic_16.sh
TEST_PROGS += test_generic_17.sh
TEST_PROGS += test_generic_18.sh
TEST_PROGS += test_batch_01.sh
TEST_PROGS += test_batch_02.sh
@@ -76,13 +77,14 @@ TEST_FILES := settings
TEST_FILES += test_common.sh
TEST_FILES += trace
TEST_GEN_PROGS_EXTENDED = kublk metadata_size
STANDALONE_UTILS := metadata_size.c
TEST_GEN_PROGS_EXTENDED = kublk metadata_size ublk_cancel_ready
STANDALONE_UTILS := metadata_size.c ublk_cancel_ready.c
LOCAL_HDRS += $(wildcard *.h)
include ../lib.mk
$(OUTPUT)/kublk: $(filter-out $(STANDALONE_UTILS),$(wildcard *.c))
$(OUTPUT)/ublk_cancel_ready: ublk_cancel_ready.c ctrl.c
check:
shellcheck -x -f gcc *.sh
tools/testing/selftests/ublk/batch.c
+10 -3
View File
@@ -410,9 +410,16 @@ static void ublk_batch_compl_commit_cmd(struct ublk_thread *t,
{
unsigned short buf_idx = user_data_to_tag(cqe->user_data);
if (op == _IOC_NR(UBLK_U_IO_PREP_IO_CMDS))
ublk_assert(cqe->res == 0);
else if (op == _IOC_NR(UBLK_U_IO_COMMIT_IO_CMDS)) {
if (op == _IOC_NR(UBLK_U_IO_PREP_IO_CMDS)) {
/* e.g. UBLK_IO_RES_ABORT after STOP_DEV: stop like a FETCH */
if (cqe->res < 0) {
ublk_dbg(UBLK_DBG_IO_CMD, "%s: prep failed %d\n",
__func__, cqe->res);
t->state |= UBLKS_T_STOPPING;
} else {
ublk_assert(cqe->res == 0);
}
} else if (op == _IOC_NR(UBLK_U_IO_COMMIT_IO_CMDS)) {
int nr_elem = user_data_to_tgt_data(cqe->user_data);
ublk_assert(cqe->res == t->commit_buf_elem_size * nr_elem);
tools/testing/selftests/ublk/ctrl.c
+268
View File
@@ -0,0 +1,268 @@
// SPDX-License-Identifier: GPL-2.0
/* ublk control commands, shared by kublk and ublk_cancel_ready */
#include "kublk.h"
static void ublk_ctrl_init_cmd(struct ublk_dev *dev,
struct io_uring_sqe *sqe,
struct ublk_ctrl_cmd_data *data)
{
struct ublksrv_ctrl_dev_info *info = &dev->dev_info;
struct ublksrv_ctrl_cmd *cmd = (struct ublksrv_ctrl_cmd *)ublk_get_sqe_cmd(sqe);
sqe->fd = dev->ctrl_fd;
sqe->opcode = IORING_OP_URING_CMD;
sqe->ioprio = 0;
if (data->flags & CTRL_CMD_HAS_BUF) {
cmd->addr = data->addr;
cmd->len = data->len;
}
if (data->flags & CTRL_CMD_HAS_DATA)
cmd->data[0] = data->data[0];
cmd->dev_id = info->dev_id;
cmd->queue_id = -1;
ublk_set_sqe_cmd_op(sqe, data->cmd_op);
io_uring_sqe_set_data(sqe, cmd);
}
int __ublk_ctrl_cmd(struct ublk_dev *dev,
struct ublk_ctrl_cmd_data *data)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret = -EINVAL;
sqe = io_uring_get_sqe(&dev->ring);
if (!sqe) {
ublk_err("%s: can't get sqe ret %d\n", __func__, ret);
return ret;
}
ublk_ctrl_init_cmd(dev, sqe, data);
ret = io_uring_submit(&dev->ring);
if (ret < 0) {
ublk_err("uring submit ret %d\n", ret);
return ret;
}
ret = io_uring_wait_cqe(&dev->ring, &cqe);
if (ret < 0) {
ublk_err("wait cqe: %s\n", strerror(-ret));
return ret;
}
io_uring_cqe_seen(&dev->ring, cqe);
return cqe->res;
}
void ublk_ctrl_deinit(struct ublk_dev *dev)
{
io_uring_queue_exit(&dev->ring);
close(dev->ctrl_fd);
free(dev);
}
struct ublk_dev *ublk_ctrl_init(void)
{
struct ublk_dev *dev = (struct ublk_dev *)calloc(1, sizeof(*dev));
struct ublksrv_ctrl_dev_info *info;
int ret;
if (!dev)
return NULL;
info = &dev->dev_info;
dev->ctrl_fd = open(CTRL_DEV, O_RDWR);
if (dev->ctrl_fd < 0) {
free(dev);
return NULL;
}
info->max_io_buf_bytes = UBLK_IO_MAX_BYTES;
ret = ublk_setup_ring(&dev->ring, UBLK_CTRL_RING_DEPTH,
UBLK_CTRL_RING_DEPTH, IORING_SETUP_SQE128);
if (ret < 0) {
ublk_err("queue_init: %s\n", strerror(-ret));
close(dev->ctrl_fd);
free(dev);
return NULL;
}
dev->nr_fds = 1;
return dev;
}
int ublk_ctrl_stop_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_STOP_DEV,
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_try_stop_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_TRY_STOP_DEV,
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_start_dev(struct ublk_dev *dev,
int daemon_pid)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_START_DEV,
.flags = CTRL_CMD_HAS_DATA,
};
dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_start_user_recovery(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_START_USER_RECOVERY,
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_END_USER_RECOVERY,
.flags = CTRL_CMD_HAS_DATA,
};
dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_add_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_ADD_DEV,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) &dev->dev_info,
.len = sizeof(struct ublksrv_ctrl_dev_info),
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_del_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_DEL_DEV,
.flags = 0,
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_get_info(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_DEV_INFO,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) &dev->dev_info,
.len = sizeof(struct ublksrv_ctrl_dev_info),
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_set_params(struct ublk_dev *dev,
struct ublk_params *params)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_SET_PARAMS,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) params,
.len = sizeof(*params),
};
params->len = sizeof(*params);
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_get_params(struct ublk_dev *dev,
struct ublk_params *params)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_PARAMS,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64)params,
.len = sizeof(*params),
};
params->len = sizeof(*params);
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_get_features(struct ublk_dev *dev,
__u64 *features)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_FEATURES,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) features,
.len = sizeof(*features),
};
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_update_size(struct ublk_dev *dev,
__u64 nr_sects)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_UPDATE_SIZE,
.flags = CTRL_CMD_HAS_DATA,
};
data.data[0] = nr_sects;
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_quiesce_dev(struct ublk_dev *dev, unsigned int timeout_ms)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_QUIESCE_DEV,
.flags = CTRL_CMD_HAS_DATA,
};
data.data[0] = timeout_ms;
return __ublk_ctrl_cmd(dev, &data);
}
int ublk_ctrl_reg_buf(struct ublk_dev *dev, void *addr, size_t size,
__u32 flags)
{
struct ublk_shmem_buf_reg buf_reg = {
.addr = (unsigned long)addr,
.len = size,
.flags = flags,
};
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_REG_BUF,
.flags = CTRL_CMD_HAS_BUF,
.addr = (unsigned long)&buf_reg,
.len = sizeof(buf_reg),
};
return __ublk_ctrl_cmd(dev, &data);
}
tools/testing/selftests/ublk/fault_inject.c
+8 -6
View File
@@ -11,7 +11,7 @@
#include "kublk.h"
struct fi_opts {
long long delay_ns;
struct __kernel_timespec delay;
bool die_during_fetch;
};
@@ -47,7 +47,8 @@ static int ublk_fault_inject_tgt_init(const struct dev_ctx *ctx,
return -ENOMEM;
}
opts->delay_ns = ctx->fault_inject.delay_us * 1000;
opts->delay.tv_sec = ctx->fault_inject.delay_us / 1000000;
opts->delay.tv_nsec = ctx->fault_inject.delay_us % 1000000 * 1000;
opts->die_during_fetch = ctx->fault_inject.die_during_fetch;
dev->private_data = opts;
@@ -85,12 +86,13 @@ static int ublk_fault_inject_queue_io(struct ublk_thread *t,
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
struct io_uring_sqe *sqe;
struct fi_opts *opts = q->dev->private_data;
struct __kernel_timespec ts = {
.tv_nsec = opts->delay_ns,
};
/*
* Pass a timespec that outlives this function, since io_uring
* reads it only when the SQE is submitted.
*/
ublk_io_alloc_sqes(t, &sqe, 1);
io_uring_prep_timeout(sqe, &ts, 1, 0);
io_uring_prep_timeout(sqe, &opts->delay, 0, 0);
sqe->user_data = build_user_data(tag, ublksrv_get_op(iod), 0, q->q_id, 1);
ublk_queued_tgt_io(t, q, tag, 1);
tools/testing/selftests/ublk/kublk.c
-271
View File
@@ -37,203 +37,6 @@ static const struct ublk_tgt_ops *ublk_find_tgt(const char *name)
return NULL;
}
static inline int ublk_setup_ring(struct io_uring *r, int depth,
int cq_depth, unsigned flags)
{
struct io_uring_params p;
memset(&p, 0, sizeof(p));
p.flags = flags | IORING_SETUP_CQSIZE;
p.cq_entries = cq_depth;
return io_uring_queue_init_params(depth, r, &p);
}
static void ublk_ctrl_init_cmd(struct ublk_dev *dev,
struct io_uring_sqe *sqe,
struct ublk_ctrl_cmd_data *data)
{
struct ublksrv_ctrl_dev_info *info = &dev->dev_info;
struct ublksrv_ctrl_cmd *cmd = (struct ublksrv_ctrl_cmd *)ublk_get_sqe_cmd(sqe);
sqe->fd = dev->ctrl_fd;
sqe->opcode = IORING_OP_URING_CMD;
sqe->ioprio = 0;
if (data->flags & CTRL_CMD_HAS_BUF) {
cmd->addr = data->addr;
cmd->len = data->len;
}
if (data->flags & CTRL_CMD_HAS_DATA)
cmd->data[0] = data->data[0];
cmd->dev_id = info->dev_id;
cmd->queue_id = -1;
ublk_set_sqe_cmd_op(sqe, data->cmd_op);
io_uring_sqe_set_data(sqe, cmd);
}
static int __ublk_ctrl_cmd(struct ublk_dev *dev,
struct ublk_ctrl_cmd_data *data)
{
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
int ret = -EINVAL;
sqe = io_uring_get_sqe(&dev->ring);
if (!sqe) {
ublk_err("%s: can't get sqe ret %d\n", __func__, ret);
return ret;
}
ublk_ctrl_init_cmd(dev, sqe, data);
ret = io_uring_submit(&dev->ring);
if (ret < 0) {
ublk_err("uring submit ret %d\n", ret);
return ret;
}
ret = io_uring_wait_cqe(&dev->ring, &cqe);
if (ret < 0) {
ublk_err("wait cqe: %s\n", strerror(-ret));
return ret;
}
io_uring_cqe_seen(&dev->ring, cqe);
return cqe->res;
}
static int ublk_ctrl_stop_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_STOP_DEV,
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_try_stop_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_TRY_STOP_DEV,
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_start_dev(struct ublk_dev *dev,
int daemon_pid)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_START_DEV,
.flags = CTRL_CMD_HAS_DATA,
};
dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_start_user_recovery(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_START_USER_RECOVERY,
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_END_USER_RECOVERY,
.flags = CTRL_CMD_HAS_DATA,
};
dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_add_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_ADD_DEV,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) &dev->dev_info,
.len = sizeof(struct ublksrv_ctrl_dev_info),
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_del_dev(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_DEL_DEV,
.flags = 0,
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_get_info(struct ublk_dev *dev)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_DEV_INFO,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) &dev->dev_info,
.len = sizeof(struct ublksrv_ctrl_dev_info),
};
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_set_params(struct ublk_dev *dev,
struct ublk_params *params)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_SET_PARAMS,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) params,
.len = sizeof(*params),
};
params->len = sizeof(*params);
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_get_params(struct ublk_dev *dev,
struct ublk_params *params)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_PARAMS,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64)params,
.len = sizeof(*params),
};
params->len = sizeof(*params);
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_get_features(struct ublk_dev *dev,
__u64 *features)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_FEATURES,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64) (uintptr_t) features,
.len = sizeof(*features),
};
return __ublk_ctrl_cmd(dev, &data);
}
static int parse_param_types(const char *arg, __u32 *types)
{
char buf[128], *save = NULL, *tok;
@@ -283,30 +86,6 @@ static void ublk_init_params_from_ctx(const struct dev_ctx *ctx,
};
}
static int ublk_ctrl_update_size(struct ublk_dev *dev,
__u64 nr_sects)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_UPDATE_SIZE,
.flags = CTRL_CMD_HAS_DATA,
};
data.data[0] = nr_sects;
return __ublk_ctrl_cmd(dev, &data);
}
static int ublk_ctrl_quiesce_dev(struct ublk_dev *dev,
unsigned int timeout_ms)
{
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_QUIESCE_DEV,
.flags = CTRL_CMD_HAS_DATA,
};
data.data[0] = timeout_ms;
return __ublk_ctrl_cmd(dev, &data);
}
static const char *ublk_dev_state_desc(struct ublk_dev *dev)
{
switch (dev->dev_info.state) {
@@ -426,38 +205,6 @@ static void ublk_ctrl_dump(struct ublk_dev *dev)
fflush(stdout);
}
static void ublk_ctrl_deinit(struct ublk_dev *dev)
{
close(dev->ctrl_fd);
free(dev);
}
static struct ublk_dev *ublk_ctrl_init(void)
{
struct ublk_dev *dev = (struct ublk_dev *)calloc(1, sizeof(*dev));
struct ublksrv_ctrl_dev_info *info = &dev->dev_info;
int ret;
dev->ctrl_fd = open(CTRL_DEV, O_RDWR);
if (dev->ctrl_fd < 0) {
free(dev);
return NULL;
}
info->max_io_buf_bytes = UBLK_IO_MAX_BYTES;
ret = ublk_setup_ring(&dev->ring, UBLK_CTRL_RING_DEPTH,
UBLK_CTRL_RING_DEPTH, IORING_SETUP_SQE128);
if (ret < 0) {
ublk_err("queue_init: %s\n", strerror(-ret));
free(dev);
return NULL;
}
dev->nr_fds = 1;
return dev;
}
static size_t __ublk_queue_cmd_buf_sz(const struct ublk_queue *q, __u16 depth)
{
size_t size = depth * (size_t)q->io_desc_size;
@@ -1283,24 +1030,6 @@ static void ublk_shmem_unregister_all(void)
shmem_count = 0;
}
static int ublk_ctrl_reg_buf(struct ublk_dev *dev, void *addr, size_t size,
__u32 flags)
{
struct ublk_shmem_buf_reg buf_reg = {
.addr = (unsigned long)addr,
.len = size,
.flags = flags,
};
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_REG_BUF,
.flags = CTRL_CMD_HAS_BUF,
.addr = (unsigned long)&buf_reg,
.len = sizeof(buf_reg),
};
return __ublk_ctrl_cmd(dev, &data);
}
/*
* Handle one client connection: receive memfd, mmap it, register
* the VA range with kernel, send back the assigned index.
tools/testing/selftests/ublk/kublk.h
+32
View File
@@ -294,6 +294,38 @@ struct ublk_dev {
extern int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io);
static inline int ublk_setup_ring(struct io_uring *r, int depth,
int cq_depth, unsigned int flags)
{
struct io_uring_params p;
memset(&p, 0, sizeof(p));
p.flags = flags | IORING_SETUP_CQSIZE;
p.cq_entries = cq_depth;
return io_uring_queue_init_params(depth, r, &p);
}
/* ctrl.c: control commands */
struct ublk_dev *ublk_ctrl_init(void);
void ublk_ctrl_deinit(struct ublk_dev *dev);
int __ublk_ctrl_cmd(struct ublk_dev *dev, struct ublk_ctrl_cmd_data *data);
int ublk_ctrl_add_dev(struct ublk_dev *dev);
int ublk_ctrl_del_dev(struct ublk_dev *dev);
int ublk_ctrl_get_info(struct ublk_dev *dev);
int ublk_ctrl_set_params(struct ublk_dev *dev, struct ublk_params *params);
int ublk_ctrl_get_params(struct ublk_dev *dev, struct ublk_params *params);
int ublk_ctrl_get_features(struct ublk_dev *dev, __u64 *features);
int ublk_ctrl_start_dev(struct ublk_dev *dev, int daemon_pid);
int ublk_ctrl_stop_dev(struct ublk_dev *dev);
int ublk_ctrl_try_stop_dev(struct ublk_dev *dev);
int ublk_ctrl_start_user_recovery(struct ublk_dev *dev);
int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid);
int ublk_ctrl_update_size(struct ublk_dev *dev, __u64 nr_sects);
int ublk_ctrl_quiesce_dev(struct ublk_dev *dev, unsigned int timeout_ms);
int ublk_ctrl_reg_buf(struct ublk_dev *dev, void *addr, size_t size,
__u32 flags);
static inline int __ublk_use_batch_io(__u64 flags)
{
return flags & UBLK_F_BATCH_IO;
tools/testing/selftests/ublk/test_generic_18.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
ERR_CODE=0
CANCEL_PROG="$(_ublk_test_top_dir)/ublk_cancel_ready"
_prep_test "generic" "start device over canceled io commands"
# STOP_DEV while kublk's UBLK_F_BATCH_IO server prepares and fetches: the
# server has to fail cleanly when PREP or FETCH is aborted, so `kublk add`
# must return either way.
race_batch_stop()
{
local id=9999 i res started=0
_have_feature "BATCH_IO" || return "$UBLK_SKIP_CODE"
for i in $(seq 1 20); do
timeout 20 "$UBLK_PROG" add -t null -q 2 -b -n "$id" > /dev/null 2>&1 &
local pid=$!
# vary where STOP_DEV lands relative to the batch PREP and FETCH
sleep "0.$((i % 10))"
"$UBLK_PROG" stop -n "$id" > /dev/null 2>&1
wait "$pid"
res=$?
"$UBLK_PROG" del -n "$id" > /dev/null 2>&1
if [ "$res" -eq 124 ]; then
echo "run $i: kublk add -b hung after STOP_DEV"
return 255
fi
[ "$res" -eq 0 ] && started=$((started + 1))
done
echo "20 runs: started $started, stopped before start $((20 - started))"
}
# race_batch_stop is above, the other modes are described in
# ublk_cancel_ready.c
for mode in stop_start partial_fetch recovery stop_restart stop_attached \
stop_live_restart race_start race_fetch race_async_fetch \
race_batch_stop; do
dmesg_before=$(dmesg | wc -l)
if [ "$mode" = race_batch_stop ]; then
race_batch_stop > "$UBLK_TMP" 2>&1
else
timeout 60 "$CANCEL_PROG" "$mode" > "$UBLK_TMP" 2>&1
fi
res=$?
msg=""
if dmesg | tail -n +"$((dmesg_before + 1))" | \
grep -q -e "BUG:" -e "Oops" -e "WARNING:"; then
msg="$mode: kernel oops/warning"
ERR_CODE=255
elif [ "$res" -eq "$UBLK_SKIP_CODE" ]; then
[ "$ERR_CODE" -eq 0 ] && ERR_CODE=$UBLK_SKIP_CODE
elif [ "$res" -ne 0 ]; then
msg="$mode: failed ($res)"
ERR_CODE=255
fi
# the output once: on failure, or always when not quiet
[ -n "$msg" ] && echo "$msg"
if [ -n "$msg" ] || [ "$UBLK_TEST_QUIET" -eq 0 ]; then
cat "$UBLK_TMP"
fi
done
_cleanup_test
_show_result $TID $ERR_CODE
tools/testing/selftests/ublk/ublk_cancel_ready.c
+957
View File
@@ -0,0 +1,957 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Bring a ublk device live while some of its fetched io commands are
* canceled.
*
* A cancel completes a fetched command and clears io->cmd, but the io
* still counts as ready. Only ubq->canceling keeps ublk_queue_rq() away
* from the NULL io->cmd. Each mode below loses ->canceling in another way:
*
* stop_start: fetch every tag, STOP_DEV before START_DEV, START_DEV.
* STOP_DEV cancels the commands of the attached server,
* so START_DEV must get -EBUSY until that server is gone.
* An unfixed kernel starts the device over them.
*
* partial_fetch: task A fetches tags 0..depth-2 and dies, so its
* commands are canceled. Task B fetches the last tag, and
* an unfixed kernel clears ->canceling when the queue gets
* ready. /dev/ublkcN stays open all the time, so
* ublk_ch_release() never resets the queue.
*
* recovery: a UBLK_F_USER_RECOVERY device with two queues loses its
* server. During recovery task Q0 fetches queue 0, which
* clears q0->canceling, then dies. An unfixed kernel still
* has ub->canceling set because queue 1 is not ready, so
* ublk_start_cancel() does not mark queue 0 again. Reads
* are issued on a CPU mapped to queue 0.
*
* In partial_fetch and recovery, START_DEV / END_USER_RECOVERY must refuse
* with -EBUSY. An unfixed kernel oopses in ublk_queue_cmd() on a NULL
* io->cmd.
*
* stop_restart: control mode. STOP_DEV on a new device, before any
* server opened it, takes no command. A server started
* afterwards has to start the device and serve I/O.
*
* stop_attached: STOP_DEV on a device whose server opened it but fetched
* nothing stops that server too: FETCH gets ABORT and
* START_DEV -EBUSY, until a new server opens the device.
*
* stop_live_restart: STOP_DEV on a live device; once its server is gone,
* a new server has to fetch and start it again. An unfixed
* ublk_ch_release() skips the reset once the disk is gone,
* so the new FETCH gets -EBUSY.
*
* race_start: STOP_DEV and START_DEV at the same time on a device whose
* server fetched every tag. START_DEV either wins, and
* reads complete (they fail once STOP_DEV removes the
* disk), or gets -EBUSY. An unfixed kernel cancels the
* commands of the live disk.
*
* race_fetch: STOP_DEV while a server opens the device and fetches,
* then START_DEV. If STOP_DEV came before the open, it must
* not take any command, so START_DEV works and every read
* succeeds; otherwise START_DEV gets -EBUSY. An unfixed
* kernel takes commands fetched after its unlock and goes
* live over them.
*
* race_async_fetch: FETCH with IOSQE_ASYNC while STOP_DEV cancels,
* then close the ring. An unfixed FETCH marks its command
* cancelable only after publishing it and dropping
* ub->mutex; a cancel in between completes a command which
* is then put on io_uring's cancelable list. Closing the
* ring walks that list: KASAN reports a use after free.
* Needs a KASAN kernel to see the bug; otherwise it must
* just not crash.
*
* A dying task is a child process which fetches and then calls exec().
* exec() cancels the task's uring_cmds before it returns, so the cancel
* is done once the child is reaped. A thread exit does not cancel them,
* and closing the ring cancels them later, from io_ring_exit_work().
*/
#include <sched.h>
#include "kublk.h"
#include "../kselftest.h"
#define NR_QUEUES 2
#define DEPTH 4
#define BUF_SIZE (64 << 10)
#define DEV_SECTORS (64 << 11) /* 64MB */
#define NR_READS (DEPTH * 2)
#define SERVE_DELAY_US 200000
#define RACE_LOOPS 50
#define ASYNC_LOOPS 300
/* one control handle per thread, the race modes send commands from two */
static __thread struct ublk_dev *ctrl_dev;
static int cdev_fd = -1;
static int dev_id = -1;
static int nr_queues;
static void *bufs[NR_QUEUES][DEPTH];
static const struct ublksrv_io_desc *iods[NR_QUEUES];
static size_t iods_len;
static struct io_uring_sqe *get_sqe(struct io_uring *ring)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
if (!sqe) {
fprintf(stderr, "out of sqes\n");
exit(KSFT_FAIL);
}
return sqe;
}
static struct ublk_dev *ctrl(void)
{
if (!ctrl_dev) {
ctrl_dev = ublk_ctrl_init();
if (!ctrl_dev) {
fprintf(stderr, "ublk_ctrl_init failed\n");
exit(KSFT_FAIL);
}
}
ctrl_dev->dev_info.dev_id = dev_id;
return ctrl_dev;
}
static void ctrl_put(void)
{
if (ctrl_dev)
ublk_ctrl_deinit(ctrl_dev);
ctrl_dev = NULL;
}
static int dev_state(void)
{
struct ublk_dev *dev = ctrl();
int ret = ublk_ctrl_get_info(dev);
return ret ? ret : dev->dev_info.state;
}
static int open_cdev(void)
{
size_t max_len = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc);
int pg = getpagesize();
char path[64];
snprintf(path, sizeof(path), "/dev/ublkc%d", dev_id);
for (int i = 0; i < 100 && cdev_fd < 0; i++) {
cdev_fd = open(path, O_RDWR);
if (cdev_fd < 0)
usleep(50000);
}
if (cdev_fd < 0)
return -errno;
/* queue q's descriptors start at q * the size for the max depth */
max_len = (max_len + pg - 1) & ~(size_t)(pg - 1);
iods_len = (DEPTH * sizeof(struct ublksrv_io_desc) + pg - 1) &
~(size_t)(pg - 1);
for (int q = 0; q < nr_queues; q++) {
void *p = mmap(NULL, iods_len, PROT_READ,
MAP_SHARED | MAP_POPULATE, cdev_fd,
UBLKSRV_CMD_BUF_OFFSET + q * max_len);
if (p == MAP_FAILED)
return -errno;
iods[q] = p;
}
return 0;
}
/* the last reference to /dev/ublkcN runs ublk_ch_release() */
static void close_cdev(void)
{
for (int q = 0; q < nr_queues; q++) {
if (iods[q])
munmap((void *)iods[q], iods_len);
iods[q] = NULL;
}
if (cdev_fd >= 0)
close(cdev_fd);
cdev_fd = -1;
}
/* ADD_DEV and SET_PARAMS, without opening /dev/ublkcN */
static int add_dev_noopen(int queues, __u64 flags)
{
struct ublk_dev *dev = ctrl();
struct ublksrv_ctrl_dev_info info = {
.nr_hw_queues = queues,
.queue_depth = DEPTH,
.max_io_buf_bytes = BUF_SIZE,
.dev_id = -1,
.flags = UBLK_F_NO_AUTO_PART_SCAN | flags,
};
struct ublk_params p = {
.types = UBLK_PARAM_TYPE_BASIC,
.basic = {
.logical_bs_shift = 9,
.physical_bs_shift = 12,
.io_opt_shift = 12,
.io_min_shift = 9,
.max_sectors = BUF_SIZE >> 9,
.dev_sectors = DEV_SECTORS,
},
};
int ret;
nr_queues = queues;
dev->dev_info = info;
ret = ublk_ctrl_add_dev(dev);
if (ret)
return ret;
dev_id = dev->dev_info.dev_id;
ret = ublk_ctrl_set_params(ctrl(), &p);
if (ret)
return ret;
for (int q = 0; q < queues; q++)
for (int i = 0; i < DEPTH; i++)
if (!bufs[q][i] &&
posix_memalign(&bufs[q][i], getpagesize(), BUF_SIZE))
return -ENOMEM;
return 0;
}
static int add_dev(int queues, __u64 flags)
{
return add_dev_noopen(queues, flags) ?: open_cdev();
}
static void cleanup(void)
{
if (dev_id < 0)
return;
ublk_ctrl_stop_dev(ctrl());
close_cdev();
ublk_ctrl_del_dev(ctrl());
dev_id = -1;
}
/* race_async_fetch sets IOSQE_ASYNC on the io commands */
static int io_cmd_sqe_flags;
static void queue_io_cmd(struct io_uring *ring, __u32 op, int q, int tag,
int res)
{
struct io_uring_sqe *sqe = get_sqe(ring);
struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd;
memset(sqe, 0, sizeof(*sqe));
sqe->fd = cdev_fd;
sqe->opcode = IORING_OP_URING_CMD;
sqe->flags = io_cmd_sqe_flags;
ublk_set_sqe_cmd_op(sqe, op);
cmd->q_id = q;
cmd->tag = tag;
cmd->result = res;
cmd->addr = (__u64)(uintptr_t)bufs[q][tag];
io_uring_sqe_set_data64(sqe, (q << 16) | tag);
}
struct async_arg {
pthread_barrier_t go, stopped;
};
/*
* FETCH every tag with IOSQE_ASYNC, racing STOP_DEV, then close the ring once
* STOP_DEV returned: that walks io_uring's list of cancelable commands.
*/
static void *race_async_fn(void *data)
{
struct async_arg *a = data;
struct io_uring ring;
int ok = !io_uring_queue_init(DEPTH, &ring, 0);
if (ok)
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, 0, tag, 0);
pthread_barrier_wait(&a->go);
if (ok)
io_uring_submit(&ring);
pthread_barrier_wait(&a->stopped);
if (ok)
io_uring_queue_exit(&ring);
return NULL;
}
/* reap @nr completions of canceled fetch commands, return how many */
static int reap_aborts(struct io_uring *ring, int nr)
{
struct __kernel_timespec ts = { .tv_sec = 2 };
struct io_uring_cqe *cqe;
int aborted = 0;
while (nr--) {
if (io_uring_wait_cqe_timeout(ring, &cqe, &ts))
break;
if (cqe->res == UBLK_IO_RES_ABORT)
aborted++;
io_uring_cqe_seen(ring, cqe);
}
return aborted;
}
/*
* A server thread fetches @nr_tags tags of queue @q, then completes each
* request after @delay_us, until its commands are aborted.
*/
struct server {
int q, first_tag, nr_tags, delay_us;
int failed; /* result which ended the serve loop, 0 if none */
pthread_t thread;
pthread_barrier_t fetched;
/* race_fetch: wait for @go and @race_delay_us, open, fetch one by one */
pthread_barrier_t go;
int race, race_delay_us;
};
static void *server_fn(void *data)
{
struct server *s = data;
struct io_uring ring;
struct io_uring_cqe *cqe;
if (io_uring_queue_init(DEPTH, &ring, 0)) {
pthread_barrier_wait(&s->fetched);
return NULL;
}
if (s->race) {
pthread_barrier_wait(&s->go);
usleep(s->race_delay_us);
if (open_cdev()) {
pthread_barrier_wait(&s->fetched);
io_uring_queue_exit(&ring);
return NULL;
}
}
for (int tag = s->first_tag; tag < s->first_tag + s->nr_tags; tag++) {
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, s->q, tag, 0);
if (s->race)
io_uring_submit(&ring);
}
io_uring_submit(&ring);
pthread_barrier_wait(&s->fetched);
while (!io_uring_wait_cqe(&ring, &cqe)) {
int tag = cqe->user_data & 0xffff;
const struct ublksrv_io_desc *iod = &iods[s->q][tag];
int res = cqe->res;
io_uring_cqe_seen(&ring, cqe);
if (res != UBLK_IO_RES_OK) {
__atomic_store_n(&s->failed, res, __ATOMIC_RELEASE);
break;
}
usleep(s->delay_us);
res = ublksrv_get_op(iod) <= UBLK_IO_OP_WRITE ?
iod->nr_sectors << 9 : 0;
queue_io_cmd(&ring, UBLK_U_IO_COMMIT_AND_FETCH_REQ, s->q, tag,
res);
io_uring_submit(&ring);
}
io_uring_queue_exit(&ring);
return NULL;
}
/* returns once the fetch commands are issued; with @race, call race_go() */
static void server_start(struct server *s)
{
pthread_barrier_init(&s->fetched, NULL, 2);
pthread_barrier_init(&s->go, NULL, 2);
pthread_create(&s->thread, NULL, server_fn, s);
if (!s->race)
pthread_barrier_wait(&s->fetched);
}
/* wait up to 5s for the serve loop of @s to end, return its result */
static int server_wait_failed(struct server *s)
{
int res = 0;
for (int i = 0; i < 500 && !res; i++) {
res = __atomic_load_n(&s->failed, __ATOMIC_ACQUIRE);
if (!res)
usleep(10000);
}
return res;
}
/*
* A task fetches @nr_tags tags of queue @q and dies. Returns once its
* commands are canceled, see the top of this file.
*/
static int fetch_and_die(int q, int first_tag, int nr_tags)
{
int status;
pid_t pid = fork();
if (pid < 0)
return -1;
if (!pid) {
struct io_uring ring;
if (io_uring_queue_init(DEPTH, &ring, 0))
_exit(1);
for (int tag = first_tag; tag < first_tag + nr_tags; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, q, tag, 0);
if (io_uring_submit(&ring) != nr_tags)
_exit(1);
execlp("true", "true", NULL);
_exit(1);
}
if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) ||
WEXITSTATUS(status))
return -1;
printf("q%d: task died with %d fetch cmds in flight\n", q, nr_tags);
return 0;
}
struct reads {
struct io_uring ring;
void *buf;
int fd, done, ok, eio, other;
};
/*
* Issue NR_READS reads at once, from @cpu if it is not negative: blk-mq
* maps the submitting CPU to the hw queue. A server holding its live
* tags for a while makes the other reads take the canceled tags.
*/
static int open_tries = 100; /* 50ms each */
static int reads_submit(struct reads *r, int cpu)
{
char path[64];
cpu_set_t set, old;
snprintf(path, sizeof(path), "/dev/ublkb%d", dev_id);
r->fd = -1;
for (int i = 0; i < open_tries && r->fd < 0; i++) {
r->fd = open(path, O_RDONLY | O_DIRECT);
if (r->fd < 0)
usleep(50000);
}
if (r->fd < 0) {
fprintf(stderr, "open %s: %s\n", path, strerror(errno));
return -1;
}
if (posix_memalign(&r->buf, 4096, NR_READS * 4096) ||
io_uring_queue_init(NR_READS, &r->ring, 0))
return -1;
for (int i = 0; i < NR_READS; i++)
io_uring_prep_read(get_sqe(&r->ring), r->fd,
r->buf + i * 4096, 4096, i * 4096);
if (cpu >= 0) {
sched_getaffinity(0, sizeof(old), &old);
CPU_ZERO(&set);
CPU_SET(cpu, &set);
sched_setaffinity(0, sizeof(set), &set);
}
io_uring_submit(&r->ring);
if (cpu >= 0)
sched_setaffinity(0, sizeof(old), &old);
return 0;
}
static void reads_reap(struct reads *r, int timeout_s)
{
struct __kernel_timespec ts = { .tv_sec = timeout_s };
struct io_uring_cqe *cqe;
while (r->done < NR_READS &&
!io_uring_wait_cqe_timeout(&r->ring, &cqe, &ts)) {
if (cqe->res == 4096)
r->ok++;
else if (cqe->res == -EIO)
r->eio++;
else
r->other++;
r->done++;
io_uring_cqe_seen(&r->ring, cqe);
}
}
static void reads_put(struct reads *r)
{
io_uring_queue_exit(&r->ring);
close(r->fd);
free(r->buf);
}
static int reads_result(struct reads *r)
{
printf("reads: %d ok, %d -EIO, %d other, %d not completed\n",
r->ok, r->eio, r->other, NR_READS - r->done);
reads_put(r);
return r->other || r->done < NR_READS ? KSFT_FAIL : KSFT_PASS;
}
static int start_dev(void)
{
int ret = ublk_ctrl_start_dev(ctrl(), getpid());
printf("START_DEV: %d\n", ret);
return ret;
}
static int expect_err(const char *what, int ret, int want)
{
if (ret == want)
return KSFT_PASS;
fprintf(stderr, "%s: %d, expected %d\n", what, ret, want);
return KSFT_FAIL;
}
/* START_DEV must fail with @want; if it went live, show what a read does */
static int start_dev_expect(int want)
{
struct reads r = {};
int ret = start_dev();
if (!ret && !reads_submit(&r, -1)) {
reads_reap(&r, 10);
reads_result(&r);
}
return expect_err("START_DEV", ret, want);
}
static int test_stop_start(void)
{
struct io_uring ring;
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
if (io_uring_queue_init(DEPTH, &ring, 0))
return KSFT_FAIL;
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, 0, tag, 0);
io_uring_submit(&ring);
/* device is ready but not started: state is UBLK_S_DEV_DEAD */
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d, canceled fetch cmds: %d/%d\n", ret,
reap_aborts(&ring, DEPTH), DEPTH);
/* STOP_DEV canceled the attached server: no start until it exits */
ret = start_dev_expect(-EBUSY);
io_uring_queue_exit(&ring);
return ret;
}
static int test_partial_fetch(void)
{
struct server b = { .q = 0, .first_tag = DEPTH - 1, .nr_tags = 1,
.delay_us = SERVE_DELAY_US };
int ret;
if (add_dev(1, 0) || fetch_and_die(0, 0, DEPTH - 1))
return KSFT_FAIL;
/* the last FETCH makes the queue ready and clears ->canceling */
server_start(&b);
ret = start_dev_expect(-EBUSY);
cleanup();
pthread_join(b.thread, NULL);
return ret;
}
static int test_stop_restart(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
struct reads r = {};
int ret;
if (add_dev_noopen(1, 0))
return KSFT_FAIL;
/* no server is attached, so there is nothing to cancel */
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
if (open_cdev())
return KSFT_FAIL;
server_start(&s);
if (start_dev()) {
ret = KSFT_FAIL;
} else if (reads_submit(&r, -1)) {
ret = KSFT_FAIL;
} else {
reads_reap(&r, 10);
ret = reads_result(&r);
if (r.ok != NR_READS)
ret = KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
return ret;
}
struct start_arg {
pthread_barrier_t go;
int ret, reads_ok;
};
static void *race_start_fn(void *data)
{
struct start_arg *a = data;
struct reads r = {};
pthread_barrier_wait(&a->go);
a->ret = ublk_ctrl_start_dev(ctrl(), getpid());
a->reads_ok = 1;
/*
* The disk may be gone already if STOP_DEV came right after, and
* reads may fail then: they only have to complete.
*/
if (!a->ret && !reads_submit(&r, -1)) {
reads_reap(&r, 5);
a->reads_ok = r.done == NR_READS;
if (a->reads_ok)
reads_put(&r);
else
reads_result(&r);
}
ctrl_put();
return NULL;
}
static int test_race_start(void)
{
int live = 0, ebusy = 0;
open_tries = 4;
for (int i = 0; i < RACE_LOOPS; i++) {
struct server s = { .q = 0, .nr_tags = DEPTH };
struct start_arg a = {};
pthread_t t;
if (add_dev(1, 0))
return KSFT_FAIL;
server_start(&s);
pthread_barrier_init(&a.go, NULL, 2);
pthread_create(&t, NULL, race_start_fn, &a);
pthread_barrier_wait(&a.go);
ublk_ctrl_stop_dev(ctrl());
pthread_join(t, NULL);
cleanup();
pthread_join(s.thread, NULL);
if (a.ret == 0)
live++;
else if (a.ret == -EBUSY)
ebusy++;
if ((a.ret && a.ret != -EBUSY) || !a.reads_ok) {
fprintf(stderr, "loop %d: START_DEV %d, reads %s\n",
i, a.ret, a.reads_ok ? "ok" : "failed");
return KSFT_FAIL;
}
}
printf("%d loops: START_DEV won %d, got -EBUSY %d\n", RACE_LOOPS,
live, ebusy);
return KSFT_PASS;
}
static int test_race_fetch(void)
{
int live = 0, ebusy = 0;
for (int i = 0; i < RACE_LOOPS; i++) {
/* vary who goes first: STOP_DEV, or the server's open */
struct server s = { .q = 0, .nr_tags = DEPTH, .race = 1,
.race_delay_us = (i % 10) * 50 };
struct reads r = {};
int ret;
if (add_dev_noopen(1, 0))
return KSFT_FAIL;
server_start(&s);
pthread_barrier_wait(&s.go);
ublk_ctrl_stop_dev(ctrl());
pthread_barrier_wait(&s.fetched);
ret = ublk_ctrl_start_dev(ctrl(), getpid());
if (!ret) {
/* nothing was taken, so every read has to succeed */
live++;
if (reads_submit(&r, -1))
return KSFT_FAIL;
reads_reap(&r, 5);
if (r.ok != NR_READS) {
fprintf(stderr, "loop %d: live, but ", i);
reads_result(&r);
return KSFT_FAIL;
}
reads_put(&r);
} else if (ret == -EBUSY) {
ebusy++;
} else {
fprintf(stderr, "loop %d: START_DEV %d\n", i, ret);
return KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
}
printf("%d loops: START_DEV worked %d, got -EBUSY %d\n", RACE_LOOPS,
live, ebusy);
return KSFT_PASS;
}
static int test_race_async_fetch(void)
{
io_cmd_sqe_flags = IOSQE_ASYNC;
for (int i = 0; i < ASYNC_LOOPS; i++) {
struct async_arg a;
pthread_t t;
if (add_dev(1, 0))
return KSFT_FAIL;
pthread_barrier_init(&a.go, NULL, 2);
pthread_barrier_init(&a.stopped, NULL, 2);
pthread_create(&t, NULL, race_async_fn, &a);
pthread_barrier_wait(&a.go);
/* vary where STOP_DEV lands relative to the FETCHes */
usleep((i % 5) * 1000);
ublk_ctrl_stop_dev(ctrl());
pthread_barrier_wait(&a.stopped);
pthread_join(t, NULL);
cleanup();
}
printf("%d loops done\n", ASYNC_LOOPS);
return KSFT_PASS;
}
/*
* The stopped server is gone: reopen /dev/ublkcN, start a new server and
* the device, and read from it.
*/
static int restart_server(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
struct reads r = {};
int ret;
close_cdev();
if (open_cdev())
return KSFT_FAIL;
server_start(&s);
usleep(200000);
if (s.failed) {
fprintf(stderr, "new server: FETCH failed %d\n", s.failed);
cleanup();
pthread_join(s.thread, NULL);
return KSFT_FAIL;
}
/* -EEXIST until the old disk is freed, e.g. after a udev probe */
for (int i = 0; i < 100; i++) {
ret = start_dev();
if (ret != -EEXIST)
break;
usleep(50000);
}
if (ret || reads_submit(&r, -1)) {
fprintf(stderr, "new server: START_DEV %d\n", ret);
ret = KSFT_FAIL;
} else {
reads_reap(&r, 10);
ret = reads_result(&r);
if (r.ok != NR_READS)
ret = KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
return ret;
}
/*
* stop_attached: STOP_DEV while a server has the device open but has
* fetched nothing stops that server too: its FETCH gets ABORT and
* START_DEV -EBUSY. Once it is gone, a new server can start the device.
*/
static int test_stop_attached(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
server_start(&s);
ret = server_wait_failed(&s);
printf("FETCH after STOP_DEV: %d\n", ret);
if (ret != UBLK_IO_RES_ABORT) {
cleanup();
pthread_join(s.thread, NULL);
return KSFT_FAIL;
}
pthread_join(s.thread, NULL);
if (expect_err("START_DEV", start_dev(), -EBUSY))
return KSFT_FAIL;
return restart_server();
}
/*
* stop_live_restart: STOP_DEV on a live device; once its server is gone,
* a new server has to fetch and start it again.
*/
static int test_stop_live_restart(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
server_start(&s);
if (expect_err("START_DEV", start_dev(), 0))
return KSFT_FAIL;
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
pthread_join(s.thread, NULL);
return restart_server();
}
/* first CPU which blk-mq maps to hw queue @q */
static int queue_cpu(int q)
{
char path[96];
FILE *f;
int cpu = -1;
snprintf(path, sizeof(path), "/sys/block/ublkb%d/mq/%d/cpu_list",
dev_id, q);
for (int i = 0; i < 100 && !(f = fopen(path, "r")); i++)
usleep(50000);
if (!f)
return -1;
if (fscanf(f, "%d", &cpu) != 1)
cpu = -1;
fclose(f);
return cpu;
}
static int test_recovery(void)
{
struct server q1 = { .q = 1, .nr_tags = DEPTH };
struct io_uring ring;
struct reads r = {};
int ret, cpu;
if (add_dev(NR_QUEUES, UBLK_F_USER_RECOVERY))
return KSFT_FAIL;
/* the first server: fetch everything, start, then die */
if (io_uring_queue_init(NR_QUEUES * DEPTH, &ring, 0))
return KSFT_FAIL;
for (int q = 0; q < NR_QUEUES; q++)
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, q, tag, 0);
io_uring_submit(&ring);
if (start_dev())
return KSFT_FAIL;
cpu = queue_cpu(0);
if (cpu < 0) {
printf("no CPU maps to queue 0, can't aim the reads\n");
return KSFT_SKIP;
}
io_uring_queue_exit(&ring);
close_cdev();
for (int i = 0; i < 100 && dev_state() != UBLK_S_DEV_QUIESCED; i++)
usleep(50000);
printf("server exited, state %d (QUIESCED is %d)\n", dev_state(),
UBLK_S_DEV_QUIESCED);
for (int i = 0; i < 100; i++) {
ret = ublk_ctrl_start_user_recovery(ctrl());
if (ret != -EBUSY)
break;
usleep(50000);
}
printf("START_USER_RECOVERY: %d\n", ret);
if (ret || open_cdev())
return KSFT_FAIL;
/* queue 0 gets ready, then its task dies before queue 1 is ready */
if (fetch_and_die(0, 0, DEPTH))
return KSFT_FAIL;
printf("reads on cpu %d, which maps to queue 0\n", cpu);
if (reads_submit(&r, cpu))
return KSFT_FAIL;
server_start(&q1);
ret = ublk_ctrl_end_user_recovery(ctrl(), getpid());
printf("END_USER_RECOVERY: %d\n", ret);
ret = expect_err("END_USER_RECOVERY", ret, -EBUSY);
/*
* Requests held back on queue 0 wait for STOP_DEV to fail them.
* Close the disk before DEL_DEV, which waits for its last reference.
*/
reads_reap(&r, 1);
ublk_ctrl_stop_dev(ctrl());
reads_reap(&r, 5);
if (reads_result(&r))
ret = KSFT_FAIL;
cleanup();
pthread_join(q1.thread, NULL);
return ret;
}
static const struct {
const char *name;
int (*fn)(void);
} modes[] = {
{ "stop_start", test_stop_start },
{ "partial_fetch", test_partial_fetch },
{ "recovery", test_recovery },
{ "stop_restart", test_stop_restart },
{ "race_start", test_race_start },
{ "race_fetch", test_race_fetch },
{ "race_async_fetch", test_race_async_fetch },
{ "stop_attached", test_stop_attached },
{ "stop_live_restart", test_stop_live_restart },
};
int main(int argc, char **argv)
{
int (*fn)(void) = NULL;
int ret;
for (int i = 0; argc == 2 && i < ARRAY_SIZE(modes); i++)
if (!strcmp(argv[1], modes[i].name))
fn = modes[i].fn;
if (!fn) {
fprintf(stderr, "usage: %s MODE, modes:", argv[0]);
for (int i = 0; i < ARRAY_SIZE(modes); i++)
fprintf(stderr, " %s", modes[i].name);
fprintf(stderr, "\n");
return KSFT_FAIL;
}
/* keep the output of a run that ends in an oops */
setvbuf(stdout, NULL, _IOLBF, 0);
if (access(CTRL_DEV, F_OK)) {
perror(CTRL_DEV);
return KSFT_SKIP;
}
printf("%s\n", argv[1]);
ret = fn();
cleanup();
ctrl_put();
return ret;
}