diff --git a/Documentation/filesystems/nfs/index.rst b/Documentation/filesystems/nfs/index.rst index a29a212b5b4d..dab5f16aaad1 100644 --- a/Documentation/filesystems/nfs/index.rst +++ b/Documentation/filesystems/nfs/index.rst @@ -7,6 +7,7 @@ NFS :maxdepth: 1 client-identifier + keyring exporting localio pnfs diff --git a/Documentation/filesystems/nfs/keyring.rst b/Documentation/filesystems/nfs/keyring.rst new file mode 100644 index 000000000000..29367e71a801 --- /dev/null +++ b/Documentation/filesystems/nfs/keyring.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================================== +The per-namespace NFS client keyring +==================================== + +The NFS client holds one keyring, named ".nfs", per network +namespace. An xprtsec=mtls mount presents the client certificate and +private key named by its cert_serial= and privkey_serial= mount +options, and the handshake links the mount's namespace .nfs keyring +into tlshd before tlshd reads those keys. Keys placed on the .nfs +keyring therefore need not grant user read permission: tlshd reaches +them as a possessor, and a tlshd in another network namespace never +possesses them. + +The keyring is owned by the owner of the network namespace's user +namespace, with KEY_POS_ALL and KEY_USR_ALL less SETATTR. It is not +charged to any quota. Keys added to it by userspace are charged to +the user that adds them. + +Finding the keyring serial number +================================= + +The .nfs keyring is not linked into any keyring that userspace +possesses, so a serial number is the only way to name it. The NFS +client registers a key type, "nfs_keyring", whose sole purpose is to +hand that serial number out. Its request_key handler runs in the +caller's context and does not upcall to /sbin/request-key. + +To obtain the serial number, request a key of type "nfs_keyring" with +the description ".nfs" and read its payload:: + + id=$(keyctl request2 nfs_keyring .nfs "" @s) + serial=$(keyctl print $id) + +The contract of the key type is: + + * The description is the string ".nfs". Any other description is + rejected with EINVAL before a key is allocated. + + * The callout info must be present. request_key() with a NULL + callout_info never invokes a handler and returns ENOKEY when no + matching key exists, so use request_key() with an empty string, or + ``keyctl request2`` rather than ``keyctl request``. The content of + the callout info is ignored. + + * The payload is the keyring serial number as decimal ASCII digits + with no terminating NUL. The return value of keyctl_read() gives + the length; a buffer of 12 bytes is sufficient. + + * add_key() with this type accepts only that payload. A key carrying + any other value is rejected with EINVAL, so a key found in the + caller's keyrings always holds the serial number of the caller's + namespace. + + * The key type carries KEY_TYPE_NET_DOMAIN. A key instantiated in one + network namespace does not answer a request made in another, so a + process that keeps its session keyring across setns() receives the + serial number of the namespace it is in at the time of the request. + +The serial number is not a secret. The keyring's own permissions +decide what a caller can do with it. + +Provisioning credentials +======================== + +Add the client certificate and private key to the .nfs keyring as +keys that grant no user read permission, then pass their serial +numbers as cert_serial= and privkey_serial= mount options. diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index e7862f35b72c..fd430718d02b 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -123,8 +123,23 @@ config PNFS_FILE_LAYOUT config PNFS_BLOCK tristate - depends on NFS_V4 && BLK_DEV_DM + +config PNFS_BLOCK_LAYOUT + bool "NFS client support for pNFS block layouts" + depends on NFS_V4 && BLOCK && BLK_DEV_DM + select PNFS_BLOCK + help + Enable support for the pNFS block-volume layout type (RFC 5663). + + If unsure, say N. + +config PNFS_SCSI_LAYOUT + bool "NFS client support for pNFS SCSI layouts" + depends on NFS_V4 && BLOCK default NFS_V4 + select PNFS_BLOCK + help + Enable suport for the pNFS SCSI layout type (RFC 8154). config PNFS_FLEXFILE_LAYOUT tristate diff --git a/fs/nfs/blocklayout/Makefile b/fs/nfs/blocklayout/Makefile index 7668a1bfb5fa..3403cb7fe201 100644 --- a/fs/nfs/blocklayout/Makefile +++ b/fs/nfs/blocklayout/Makefile @@ -4,4 +4,5 @@ # obj-$(CONFIG_PNFS_BLOCK) += blocklayoutdriver.o -blocklayoutdriver-y += blocklayout.o dev.o extent_tree.o rpc_pipefs.o +blocklayoutdriver-y += blocklayout.o dev.o extent_tree.o +blocklayoutdriver-$(CONFIG_PNFS_BLOCK_LAYOUT) += rpc_pipefs.o diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index d54a141a89b3..82873ff370ef 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -470,18 +470,6 @@ static struct pnfs_layout_hdr *__bl_alloc_layout_hdr(struct inode *inode, return &bl->bl_layout; } -static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode, - gfp_t gfp_flags) -{ - return __bl_alloc_layout_hdr(inode, gfp_flags, false); -} - -static struct pnfs_layout_hdr *sl_alloc_layout_hdr(struct inode *inode, - gfp_t gfp_flags) -{ - return __bl_alloc_layout_hdr(inode, gfp_flags, true); -} - static void bl_free_lseg(struct pnfs_layout_segment *lseg) { dprintk("%s enter\n", __func__); @@ -859,7 +847,8 @@ bl_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) if (pgio->pg_lseg && test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) { - pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg); + pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg, + NULL); pnfs_set_lo_fail(pgio->pg_lseg); nfs_pageio_reset_read_mds(pgio); } @@ -921,7 +910,8 @@ bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) if (pgio->pg_lseg && test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) { - pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg); + pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg, + NULL); pnfs_set_lo_fail(pgio->pg_lseg); nfs_pageio_reset_write_mds(pgio); } @@ -954,6 +944,13 @@ static const struct nfs_pageio_ops bl_pg_write_ops = { .pg_cleanup = pnfs_generic_pg_cleanup, }; +#ifdef CONFIG_PNFS_BLOCK_LAYOUT +static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode, + gfp_t gfp_flags) +{ + return __bl_alloc_layout_hdr(inode, gfp_flags, false); +} + static struct pnfs_layoutdriver_type blocklayout_type = { .id = LAYOUT_BLOCK_VOLUME, .name = "LAYOUT_BLOCK_VOLUME", @@ -978,6 +975,48 @@ static struct pnfs_layoutdriver_type blocklayout_type = { .sync = pnfs_generic_sync, }; +static int __init pnfs_register_blocklayout(void) +{ + int ret; + + ret = bl_init_pipefs(); + if (ret) + return ret; + + ret = pnfs_register_layoutdriver(&blocklayout_type); + if (ret) { + bl_cleanup_pipefs(); + return ret; + } + + return 0; +} + +static void __exit pnfs_unregister_blocklayout(void) +{ + pnfs_unregister_layoutdriver(&blocklayout_type); + bl_cleanup_pipefs(); +} + +MODULE_ALIAS("nfs-layouttype4-3"); +#else +static int __init pnfs_register_blocklayout(void) +{ + return 0; +} + +static void __exit pnfs_unregister_blocklayout(void) +{ +} +#endif /* CONFIG_PNFS_BLOCK_LAYOUT */ + +#ifdef CONFIG_PNFS_SCSI_LAYOUT +static struct pnfs_layout_hdr *sl_alloc_layout_hdr(struct inode *inode, + gfp_t gfp_flags) +{ + return __bl_alloc_layout_hdr(inode, gfp_flags, true); +} + static struct pnfs_layoutdriver_type scsilayout_type = { .id = LAYOUT_SCSI, .name = "LAYOUT_SCSI", @@ -1002,6 +1041,28 @@ static struct pnfs_layoutdriver_type scsilayout_type = { .sync = pnfs_generic_sync, }; +static int __init pnfs_register_scsilayout(void) +{ + return pnfs_register_layoutdriver(&scsilayout_type); +} + +static void __exit pnfs_unregister_scsilayout(void) +{ + pnfs_unregister_layoutdriver(&scsilayout_type); +} + +MODULE_ALIAS("nfs-layouttype4-5"); +#else +static int __init pnfs_register_scsilayout(void) +{ + return 0; +} + +static void __exit pnfs_unregister_scsilayout(void) +{ +} +#endif /* CONFIG_PNFS_SCSI_LAYOUT */ + static int __init nfs4blocklayout_init(void) { @@ -1009,25 +1070,17 @@ static int __init nfs4blocklayout_init(void) dprintk("%s: NFSv4 Block Layout Driver Registering...\n", __func__); - ret = bl_init_pipefs(); + ret = pnfs_register_blocklayout(); if (ret) - goto out; + return ret; - ret = pnfs_register_layoutdriver(&blocklayout_type); - if (ret) - goto out_cleanup_pipe; + ret = pnfs_register_scsilayout(); + if (ret) { + pnfs_unregister_blocklayout(); + return ret; + } - ret = pnfs_register_layoutdriver(&scsilayout_type); - if (ret) - goto out_unregister_block; return 0; - -out_unregister_block: - pnfs_unregister_layoutdriver(&blocklayout_type); -out_cleanup_pipe: - bl_cleanup_pipefs(); -out: - return ret; } static void __exit nfs4blocklayout_exit(void) @@ -1035,13 +1088,9 @@ static void __exit nfs4blocklayout_exit(void) dprintk("%s: NFSv4 Block Layout Driver Unregistering...\n", __func__); - pnfs_unregister_layoutdriver(&scsilayout_type); - pnfs_unregister_layoutdriver(&blocklayout_type); - bl_cleanup_pipefs(); + pnfs_unregister_scsilayout(); + pnfs_unregister_blocklayout(); } -MODULE_ALIAS("nfs-layouttype4-3"); -MODULE_ALIAS("nfs-layouttype4-5"); - module_init(nfs4blocklayout_init); module_exit(nfs4blocklayout_exit); diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index c926b7e43827..00462e6affb1 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -15,6 +15,7 @@ #define NFSDBG_FACILITY NFSDBG_PNFS_LD +#ifdef CONFIG_PNFS_SCSI_LAYOUT static void bl_unregister_scsi(struct pnfs_block_dev *dev) { struct block_device *bdev = file_bdev(dev->bdev_file); @@ -45,6 +46,16 @@ static bool bl_register_scsi(struct pnfs_block_dev *dev) trace_bl_pr_key_reg(bdev, dev->pr_key); return true; } +#else +static void bl_unregister_scsi(struct pnfs_block_dev *dev) +{ +} + +static bool bl_register_scsi(struct pnfs_block_dev *dev) +{ + return false; +} +#endif /* CONFIG_PNFS_SCSI_LAYOUT */ static void bl_unregister_dev(struct pnfs_block_dev *dev) { @@ -292,7 +303,7 @@ static int bl_parse_deviceid(struct nfs_server *server, struct pnfs_block_dev *d, struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask); - +#ifdef CONFIG_PNFS_BLOCK_LAYOUT static int bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d, struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) @@ -320,7 +331,17 @@ bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d, file_bdev(bdev_file)->bd_disk->disk_name); return 0; } +#else +static int +bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + dprintk("unsupported volume type: %d\n", PNFS_BLOCK_VOLUME_SIMPLE); + return -EIO; +} +#endif /* CONFIG_PNFS_BLOCK_LAYOUT */ +#ifdef CONFIG_PNFS_SCSI_LAYOUT static bool bl_validate_designator(struct pnfs_block_volume *v) { @@ -449,6 +470,15 @@ out_blkdev_put: d->bdev_file = NULL; return error; } +#else +static int +bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + dprintk("unsupported volume type: %d\n", PNFS_BLOCK_VOLUME_SCSI); + return -EIO; +} +#endif /* CONFIG_PNFS_SCSI_LAYOUT */ static int bl_parse_slice(struct nfs_server *server, struct pnfs_block_dev *d, diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index 3fb10c8e4271..9d836cb72e78 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -292,7 +292,7 @@ static u32 initiate_file_draining(struct nfs_client *clp, switch (pnfs_mark_matching_lsegs_return(lo, &free_me_list, &args->cbl_range, be32_to_cpu(args->cbl_stateid.seqid), - args->cbl_layoutchanged)) { + args->cbl_layoutchanged, NULL)) { case 0: case -EBUSY: /* There are layout segments that need to be returned */ @@ -391,7 +391,26 @@ __be32 nfs4_callback_devicenotify(void *argp, void *resp, if (!ld) continue; } - nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id); + /* + * Bump the epoch before touching the cache so a + * GETDEVICEINFO already in flight can detect that it + * predates the notification. A referenced DELETE may be + * racing revocation, so defer it to the state manager -- + * this thread cannot issue fore-channel RPCs. + */ + nfs4_deviceid_bump_change_epoch(cps->clp); + if (dev->cbd_notify_type == NOTIFY_DEVICEID4_CHANGE) { + nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id); + pnfs_layout_reresolve_deviceid_byclid(cps->clp, ld, + &dev->cbd_dev_id, + dev->cbd_immediate); + } else if (pnfs_layout_deviceid_referenced_byclid(cps->clp, + ld, &dev->cbd_dev_id)) { + pnfs_deviceid_delete_mark(cps->clp, ld, + &dev->cbd_dev_id); + } else { + nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id); + } } pnfs_put_layoutdriver(ld); out: diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 88e1af0fda01..5a1f05469268 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -271,6 +271,13 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp, if (n == 0) goto out; + /* sanity check the count against the remaining stream */ + if (n > xdr_stream_remaining(xdr) / + ((4 * sizeof(uint32_t)) + NFS4_DEVICEID4_SIZE)) { + status = htonl(NFS4ERR_BADXDR); + goto out; + } + args->devs = kmalloc_objs(*args->devs, n); if (!args->devs) { status = htonl(NFS4ERR_DELAY); @@ -312,7 +319,7 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp, memcpy(dev->cbd_dev_id.data, p, NFS4_DEVICEID4_SIZE); p += XDR_QUADLEN(NFS4_DEVICEID4_SIZE); - if (dev->cbd_layout_type == NOTIFY_DEVICEID4_CHANGE) { + if (dev->cbd_notify_type == NOTIFY_DEVICEID4_CHANGE) { p = xdr_inline_decode(xdr, sizeof(uint32_t)); if (unlikely(p == NULL)) { status = htonl(NFS4ERR_BADXDR); diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 60386330aeec..ea99b3c1057a 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -191,6 +191,13 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_principal = "*"; clp->cl_xprtsec = cl_init->xprtsec; + /* + * Every client in a namespace names the same keyring, so + * nfs_match_client() does not compare keyring_serial. + */ + if (clp->cl_xprtsec.policy == RPC_XPRTSEC_TLS_X509) + clp->cl_xprtsec.keyring_serial = + key_serial(nfs_net_keyring(clp->cl_net)); return clp; error_cleanup: @@ -256,7 +263,7 @@ static void pnfs_init_server(struct nfs_server *server) */ void nfs_free_client(struct nfs_client *clp) { - nfs_localio_disable_client(clp); + nfs_localio_free_client(clp); /* -EIO all pending I/O */ if (!IS_ERR(clp->cl_rpcclient)) @@ -549,7 +556,7 @@ int nfs_create_rpc_client(struct nfs_client *clp, .version = clp->rpc_ops->version, .authflavor = flavor, .cred = cl_init->cred, - .xprtsec = cl_init->xprtsec, + .xprtsec = clp->cl_xprtsec, .connect_timeout = cl_init->connect_timeout, .reconnect_timeout = cl_init->reconnect_timeout, }; @@ -1295,7 +1302,8 @@ void nfs_clients_init(struct net *net) INIT_LIST_HEAD(&nn->nfs_volume_list); #if IS_ENABLED(CONFIG_NFS_V4) idr_init(&nn->cb_ident_idr); - INIT_LIST_HEAD(&nn->nfs4_data_server_cache); + for (int i = 0; i < NFS4_DS_CACHE_HASH_SIZE; i++) + INIT_HLIST_HEAD(&nn->nfs4_data_server_cache[i]); spin_lock_init(&nn->nfs4_data_server_lock); #endif /* CONFIG_NFS_V4 */ spin_lock_init(&nn->nfs_client_lock); @@ -1315,7 +1323,8 @@ void nfs_clients_exit(struct net *net) WARN_ON_ONCE(!list_empty(&nn->nfs_client_list)); WARN_ON_ONCE(!list_empty(&nn->nfs_volume_list)); #if IS_ENABLED(CONFIG_NFS_V4) - WARN_ON_ONCE(!list_empty(&nn->nfs4_data_server_cache)); + for (int i = 0; i < NFS4_DS_CACHE_HASH_SIZE; i++) + WARN_ON_ONCE(!hlist_empty(&nn->nfs4_data_server_cache[i])); #endif /* CONFIG_NFS_V4 */ } diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index ab3e441464a6..ad774316f716 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -934,8 +934,7 @@ nfs_mark_return_if_closed_delegation(struct nfs_server *server, { struct inode *inode; - if (!list_empty_careful(&server->delegations_return) || - test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) + if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) return; spin_lock(&delegation->lock); inode = delegation->inode; diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 354f986e60c4..08a940e63152 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1935,7 +1935,7 @@ static int nfs_dentry_delete(const struct dentry *dentry) } /* Ensure that we revalidate inode->i_nlink */ -static void nfs_drop_nlink(struct inode *inode, unsigned long gencount) +void nfs_drop_nlink(struct inode *inode, unsigned long gencount) { struct nfs_inode *nfsi = NFS_I(inode); @@ -1949,6 +1949,7 @@ static void nfs_drop_nlink(struct inode *inode, unsigned long gencount) NFS_INO_INVALID_NLINK); spin_unlock(&inode->i_lock); } +EXPORT_SYMBOL_GPL(nfs_drop_nlink); /* * Called when the dentry loses inode. @@ -2490,6 +2491,8 @@ struct dentry *nfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir, trace_nfs_mkdir_enter(dir, dentry); ret = NFS_PROTO(dir)->mkdir(dir, dentry, &attr); trace_nfs_mkdir_exit(dir, dentry, PTR_ERR_OR_ZERO(ret)); + if (IS_ERR(ret)) + d_drop(dentry); return ret; } EXPORT_SYMBOL_GPL(nfs_mkdir); @@ -2554,7 +2557,6 @@ EXPORT_SYMBOL_GPL(nfs_rmdir); static int nfs_safe_remove(struct dentry *dentry) { struct inode *dir = d_inode(dentry->d_parent); - struct inode *inode = d_inode(dentry); int error = -EBUSY; dfprintk(VFS, "NFS: safe_remove(%pd2)\n", dentry); @@ -2566,14 +2568,7 @@ static int nfs_safe_remove(struct dentry *dentry) } trace_nfs_remove_enter(dir, dentry); - if (inode != NULL) { - unsigned long gencount = READ_ONCE(NFS_I(inode)->attr_gencount); - - error = NFS_PROTO(dir)->remove(dir, dentry); - if (error == 0) - nfs_drop_nlink(inode, gencount); - } else - error = NFS_PROTO(dir)->remove(dir, dentry); + error = NFS_PROTO(dir)->remove(dir, dentry); if (error == -ENOENT) nfs_dentry_handle_enoent(dentry); trace_nfs_remove_exit(dir, dentry, error); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index ccafdc1ce64d..a3c6e8f4ea06 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -145,11 +145,55 @@ static void nfs_direct_file_adjust_size_locked(struct inode *inode, } } -static void nfs_direct_release_pages(struct page **pages, unsigned int npages) +static void nfs_direct_release_pages(struct page **pages, unsigned int npages, + bool pinned) { - unsigned int i; - for (i = 0; i < npages; i++) - put_page(pages[i]); + if (pinned) + unpin_user_pages(pages, npages); +} + +static ssize_t nfs_direct_extract_pages(struct nfs_direct_req *dreq, + struct iov_iter *iter, + size_t size, loff_t *pos, + struct list_head *list) +{ + bool pinned = iov_iter_extract_will_pin(iter); + struct page **pagevec = NULL; + ssize_t result, bytes = 0; + int err = 0; + unsigned int npages, i; + size_t pgbase; + + result = iov_iter_extract_pages(iter, &pagevec, size, ~0U, 0, &pgbase); + if (result <= 0) + return result; + + npages = (result + pgbase + PAGE_SIZE - 1) >> PAGE_SHIFT; + for (i = 0; i < npages; i++) { + struct nfs_page *req; + unsigned int req_len = min_t(size_t, result - bytes, PAGE_SIZE - pgbase); + + req = nfs_page_create_from_page(dreq->ctx, pagevec[i], + pinned, pgbase, *pos, + req_len); + if (IS_ERR(req)) { + err = PTR_ERR(req); + break; + } + + list_add_tail(&req->wb_list, list); + pgbase = 0; + bytes += req_len; + *pos += req_len; + } + + if (i < npages) { + iov_iter_revert(iter, result - bytes); + nfs_direct_release_pages(pagevec + i, npages - i, pinned); + } + + kvfree(pagevec); + return bytes ? bytes : err; } void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo, @@ -288,13 +332,7 @@ out_put: static void nfs_read_sync_pgio_error(struct list_head *head, int error) { - struct nfs_page *req; - - while (!list_empty(head)) { - req = nfs_list_entry(head->next); - nfs_list_remove_request(req); - nfs_release_request(req); - } + nfs_release_request_list(head); } static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr) @@ -326,6 +364,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, ssize_t result = -EINVAL; size_t requested_bytes = 0; size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE); + LIST_HEAD(nfs_page_list); nfs_pageio_init_read(&desc, dreq->inode, false, &nfs_direct_read_completion_ops); @@ -334,40 +373,23 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, inode_dio_begin(inode); while (iov_iter_count(iter)) { - struct page **pagevec; - size_t bytes; - size_t pgbase; - unsigned npages, i; - - result = iov_iter_get_pages_alloc2(iter, &pagevec, - rsize, &pgbase); + result = nfs_direct_extract_pages(dreq, iter, rsize, &pos, &nfs_page_list); if (result < 0) break; - - bytes = result; - npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; - for (i = 0; i < npages; i++) { - struct nfs_page *req; - unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); - /* XXX do we need to do the eof zeroing found in async_filler? */ - req = nfs_page_create_from_page(dreq->ctx, pagevec[i], - pgbase, pos, req_len); - if (IS_ERR(req)) { - result = PTR_ERR(req); - break; - } + + while (!list_empty(&nfs_page_list)) { + struct nfs_page *req = nfs_list_entry(nfs_page_list.next); + size_t req_len = req->wb_bytes; + + nfs_list_remove_request(req); if (!nfs_pageio_add_request(&desc, req)) { result = desc.pg_error; nfs_release_request(req); + nfs_release_request_list(&nfs_page_list); break; } - pgbase = 0; - bytes -= req_len; requested_bytes += req_len; - pos += req_len; } - nfs_direct_release_pages(pagevec, npages); - kvfree(pagevec); if (result < 0) break; } @@ -858,6 +880,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, ssize_t result = 0; size_t requested_bytes = 0; size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE); + LIST_HEAD(nfs_page_list); bool defer = false; trace_nfs_direct_write_schedule_iovec(dreq); @@ -870,53 +893,32 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, NFS_I(inode)->write_io += iov_iter_count(iter); while (iov_iter_count(iter)) { - struct page **pagevec; - size_t bytes; - size_t pgbase; - unsigned npages, i; - - result = iov_iter_get_pages_alloc2(iter, &pagevec, - wsize, &pgbase); + result = nfs_direct_extract_pages(dreq, iter, wsize, &pos, &nfs_page_list); if (result < 0) break; - bytes = result; - npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; - for (i = 0; i < npages; i++) { - struct nfs_page *req; - unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); - - req = nfs_page_create_from_page(dreq->ctx, pagevec[i], - pgbase, pos, req_len); - if (IS_ERR(req)) { - result = PTR_ERR(req); - break; - } - - if (desc.pg_error < 0) { - nfs_free_request(req); - result = desc.pg_error; - break; - } - - pgbase = 0; - bytes -= req_len; - requested_bytes += req_len; - pos += req_len; + while (!list_empty(&nfs_page_list)) { + struct nfs_page *req = nfs_list_entry(nfs_page_list.next); + size_t req_len = req->wb_bytes; + nfs_list_remove_request(req); if (defer) { nfs_mark_request_commit(req, NULL, &cinfo, 0); + requested_bytes += req_len; continue; } nfs_lock_request(req); - if (nfs_pageio_add_request(&desc, req)) + if (nfs_pageio_add_request(&desc, req)) { + requested_bytes += req_len; continue; + } /* Exit on hard errors */ if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) { result = desc.pg_error; nfs_unlock_and_release_request(req); + nfs_release_request_list(&nfs_page_list); break; } @@ -927,11 +929,10 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, spin_unlock(&dreq->lock); nfs_unlock_request(req); nfs_mark_request_commit(req, NULL, &cinfo, 0); + requested_bytes += req_len; desc.pg_error = 0; defer = true; } - nfs_direct_release_pages(pagevec, npages); - kvfree(pagevec); if (result < 0) break; } diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index 72e20b56fbc7..53808e42546a 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -55,12 +55,13 @@ static loff_t filelayout_get_dense_offset(struct nfs4_filelayout_segment *flseg, loff_t offset) { - u32 stripe_width = flseg->stripe_unit * flseg->dsaddr->stripe_count; + u64 stripe_width = (u64)flseg->stripe_unit * + flseg->dsaddr->stripe_count; u64 stripe_no; u32 rem; offset -= flseg->pattern_offset; - stripe_no = div_u64(offset, stripe_width); + stripe_no = div64_u64(offset, stripe_width); div_u64_rem(offset, flseg->stripe_unit, &rem); return stripe_no * flseg->stripe_unit + rem; @@ -186,7 +187,7 @@ static int filelayout_async_handle_error(struct rpc_task *task, dprintk("%s DS connection error %d\n", __func__, task->tk_status); nfs4_mark_deviceid_unavailable(devid); - pnfs_error_mark_layout_for_return(inode, lseg); + pnfs_error_mark_layout_for_return(inode, lseg, NULL); pnfs_set_lo_fail(lseg); rpc_wake_up(&tbl->slot_tbl_waitq); fallthrough; @@ -796,7 +797,7 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, unsigned int size; u64 p_stripe, r_stripe; u32 stripe_offset; - u64 segment_offset = pgio->pg_lseg->pls_range.offset; + u64 pattern_offset = FILELAYOUT_LSEG(pgio->pg_lseg)->pattern_offset; u32 stripe_unit = FILELAYOUT_LSEG(pgio->pg_lseg)->stripe_unit; /* calls nfs_generic_pg_test */ @@ -808,8 +809,8 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, /* see if req and prev are in the same stripe */ if (prev) { - p_stripe = (u64)req_offset(prev) - segment_offset; - r_stripe = (u64)req_offset(req) - segment_offset; + p_stripe = (u64)req_offset(prev) - pattern_offset; + r_stripe = (u64)req_offset(req) - pattern_offset; do_div(p_stripe, stripe_unit); do_div(r_stripe, stripe_unit); @@ -818,7 +819,7 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, } /* calculate remaining bytes in the current stripe */ - div_u64_rem((u64)req_offset(req) - segment_offset, + div_u64_rem((u64)req_offset(req) - pattern_offset, stripe_unit, &stripe_offset); WARN_ON_ONCE(stripe_offset > stripe_unit); @@ -856,7 +857,7 @@ fl_pnfs_update_layout(struct inode *ino, status = filelayout_check_deviceid(lo, fl, gfp_flags); if (status) { - pnfs_error_mark_layout_for_return(ino, lseg); + pnfs_error_mark_layout_for_return(ino, lseg, NULL); pnfs_set_lo_fail(lseg); pnfs_put_lseg(lseg); lseg = NULL; diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index 88bc79ec3459..5995f32f07ec 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -177,27 +177,14 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr); /* If DS was already in cache, free ds addrs */ - while (!list_empty(&dsaddrs)) { - da = list_first_entry(&dsaddrs, - struct nfs4_pnfs_ds_addr, - da_node); - list_del_init(&da->da_node); - kfree(da->da_remotestr); - kfree(da); - } + nfs4_pnfs_ds_addr_list_free(&dsaddrs); } folio_put(scratch); return dsaddr; out_err_drain_dsaddrs: - while (!list_empty(&dsaddrs)) { - da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr, - da_node); - list_del_init(&da->da_node); - kfree(da->da_remotestr); - kfree(da); - } + nfs4_pnfs_ds_addr_list_free(&dsaddrs); out_err_free_deviceid: nfs4_fl_free_deviceid(dsaddr); /* stripe_indicies was part of dsaddr */ @@ -280,7 +267,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx) goto out_test_devid; status = nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo, - dataserver_retrans, 4, + dataserver_retrans, 0, 4, s->nfs_client->cl_minorversion, true); if (status) { nfs4_mark_deviceid_unavailable(devid); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 7fe8b91fa47c..9492d6bc1d1a 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -44,10 +44,11 @@ static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, struct nfs42_layoutstat_devinfo *devinfo, + struct nfs4_ff_layoutstat_priv *priv, int dev_limit, enum nfs4_ff_op_type type); static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr, const struct nfs42_layoutstat_devinfo *devinfo, - struct nfs4_ff_layout_ds_stripe *dss_info); + struct nfs4_ff_layoutstat_priv *priv); static struct pnfs_layout_hdr * ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) @@ -285,8 +286,8 @@ static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(u32 dss_count, mirror->dss_count = dss_count; mirror->dss = - kzalloc_objs(struct nfs4_ff_layout_ds_stripe, dss_count, - gfp_flags); + kvzalloc_objs(struct nfs4_ff_layout_ds_stripe, dss_count, + gfp_flags); if (mirror->dss == NULL) { kfree(mirror); return NULL; @@ -312,10 +313,12 @@ static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror) cred = rcu_access_pointer(mirror->dss[dss_id].rw_cred); put_cred(cred); nfs_close_local_fh(&mirror->dss[dss_id].nfl); - nfs4_ff_layout_put_deviceid(mirror->dss[dss_id].mirror_ds); + /* the last reference to the mirror is gone; no concurrency */ + nfs4_ff_layout_put_deviceid(rcu_dereference_protected( + mirror->dss[dss_id].mirror_ds, 1)); } - kfree(mirror->dss); + kvfree(mirror->dss); kfree(mirror); } @@ -636,9 +639,12 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, if (!p) goto out_sort_mirrors; fls->flags = be32_to_cpup(p); - if (fls->flags & FF_FLAGS_NO_IO_THRU_MDS) + if (fls->flags & FF_FLAGS_NO_IO_THRU_MDS) { set_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, &FF_LAYOUT_FROM_HDR(lh)->flags); + /* Outlives this layout hdr; see NFS_INO_NO_IO_THRU_MDS */ + nfs_set_no_io_thru_mds(lh->plh_inode); + } p = xdr_inline_decode(&stream, 4); if (!p) @@ -857,33 +863,17 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task, spin_unlock(&mirror->lock); } -static void -ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) -{ - struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); - - if (devid) - nfs4_mark_deviceid_unavailable(devid); -} - -static void -ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) -{ - struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); - - if (devid) - nfs4_mark_deviceid_available(devid); -} - -static struct nfs4_pnfs_ds * +static struct nfs4_ff_layout_ds * ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, u32 start_idx, u32 *best_idx, - u32 offset, u32 *dss_id, + u64 offset, u32 *dss_id, bool check_device) { struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); struct nfs4_ff_layout_mirror *mirror; - struct nfs4_pnfs_ds *ds = ERR_PTR(-EAGAIN); + struct nfs4_ff_layout_ds *mirror_ds; + struct nfs4_ff_layout_ds *ret = ERR_PTR(-EAGAIN); + struct nfs4_pnfs_ds *ds; u32 idx; /* mirrors are initially sorted by efficiency */ @@ -893,74 +883,97 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, fls->stripe_unit, fls->mirror_array[idx]->dss_count, offset); - ds = nfs4_ff_layout_prepare_ds(lseg, mirror, *dss_id, false); - if (IS_ERR(ds)) + mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, + *dss_id); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, + *dss_id, OP_READ); + if (IS_ERR(ds)) { + nfs4_ff_layout_put_deviceid(mirror_ds); + ret = ERR_CAST(ds); continue; + } if (check_device && - nfs4_test_deviceid_unavailable(&mirror->dss[*dss_id].mirror_ds->id_node)) { + nfs4_test_deviceid_unavailable(&mirror_ds->id_node)) { + nfs4_ff_layout_put_deviceid(mirror_ds); // reinitialize the error state in case if this is the last iteration - ds = ERR_PTR(-EINVAL); + ret = ERR_PTR(-EINVAL); continue; } *best_idx = idx; - break; + return mirror_ds; } - return ds; + return ret; } -static struct nfs4_pnfs_ds * +static struct nfs4_ff_layout_ds * ff_layout_choose_any_ds_for_read(struct pnfs_layout_segment *lseg, u32 start_idx, u32 *best_idx, - u32 offset, u32 *dss_id) + u64 offset, u32 *dss_id) { return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, offset, dss_id, false); } -static struct nfs4_pnfs_ds * +static struct nfs4_ff_layout_ds * ff_layout_choose_valid_ds_for_read(struct pnfs_layout_segment *lseg, u32 start_idx, u32 *best_idx, - u32 offset, u32 *dss_id) + u64 offset, u32 *dss_id) { return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, offset, dss_id, true); } -static struct nfs4_pnfs_ds * +static struct nfs4_ff_layout_ds * ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg, u32 start_idx, u32 *best_idx, - u32 offset, u32 *dss_id) + u64 offset, u32 *dss_id) { - struct nfs4_pnfs_ds *ds; + struct nfs4_ff_layout_ds *mirror_ds; - ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, best_idx, - offset, dss_id); - if (!IS_ERR(ds)) - return ds; + mirror_ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, + best_idx, offset, + dss_id); + if (!IS_ERR(mirror_ds)) + return mirror_ds; return ff_layout_choose_any_ds_for_read(lseg, start_idx, best_idx, offset, dss_id); } -static struct nfs4_pnfs_ds * +static struct nfs4_ff_layout_ds * ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio, u32 *best_idx, - u32 offset, + u64 offset, u32 *dss_id) { struct pnfs_layout_segment *lseg = pgio->pg_lseg; - struct nfs4_pnfs_ds *ds; + struct nfs4_ff_layout_ds *mirror_ds; - ds = ff_layout_choose_best_ds_for_read(lseg, pgio->pg_mirror_idx, - best_idx, offset, dss_id); - if (!IS_ERR(ds) || !pgio->pg_mirror_idx) - return ds; + mirror_ds = ff_layout_choose_best_ds_for_read(lseg, + pgio->pg_mirror_idx, + best_idx, offset, dss_id); + if (!IS_ERR(mirror_ds) || !pgio->pg_mirror_idx) + return mirror_ds; return ff_layout_choose_best_ds_for_read(lseg, 0, best_idx, offset, dss_id); } +/* + * Wait before retrying pg_init. Returns false, with pgio->pg_error set, + * if the task was killed or is exiting and must not retry. + */ +static bool ff_layout_pg_retry_wait(struct nfs_pageio_descriptor *pgio) +{ + if (fatal_signal_pending(current) || nfs_current_task_exiting()) { + pgio->pg_error = -ERESTARTSYS; + return false; + } + ssleep(1); + return true; +} + static void ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req, @@ -995,9 +1008,8 @@ ff_layout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, { unsigned int size; u64 p_stripe, r_stripe; - u32 stripe_offset; - u64 segment_offset = pgio->pg_lseg->pls_range.offset; - u32 stripe_unit = FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit; + u64 stripe_offset; + u64 stripe_unit = FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit; /* calls nfs_generic_pg_test */ size = pnfs_generic_pg_test(pgio, prev, req); @@ -1008,23 +1020,23 @@ ff_layout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, /* see if req and prev are in the same stripe */ if (prev) { - p_stripe = (u64)req_offset(prev) - segment_offset; - r_stripe = (u64)req_offset(req) - segment_offset; - do_div(p_stripe, stripe_unit); - do_div(r_stripe, stripe_unit); + p_stripe = (u64)req_offset(prev); + r_stripe = (u64)req_offset(req); + p_stripe = div64_u64(p_stripe, stripe_unit); + r_stripe = div64_u64(r_stripe, stripe_unit); if (p_stripe != r_stripe) return 0; } /* calculate remaining bytes in the current stripe */ - div_u64_rem((u64)req_offset(req) - segment_offset, + div64_u64_rem((u64)req_offset(req), stripe_unit, &stripe_offset); WARN_ON_ONCE(stripe_offset > stripe_unit); if (stripe_offset >= stripe_unit) return 0; - return min(stripe_unit - (unsigned int)stripe_offset, size); + return min_t(u64, stripe_unit - stripe_offset, size); } static void @@ -1032,8 +1044,7 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) { struct nfs_pgio_mirror *pgm; - struct nfs4_ff_layout_mirror *mirror; - struct nfs4_pnfs_ds *ds; + struct nfs4_ff_layout_ds *mirror_ds; u32 ds_idx, dss_id; if (NFS_SERVER(pgio->pg_inode)->flags & @@ -1055,20 +1066,21 @@ retry: /* Reset wb_nio, since getting layout segment was successful */ req->wb_nio = 0; - ds = ff_layout_get_ds_for_read(pgio, &ds_idx, - req_offset(req), &dss_id); - if (IS_ERR(ds)) { + mirror_ds = ff_layout_get_ds_for_read(pgio, &ds_idx, + req_offset(req), &dss_id); + if (IS_ERR(mirror_ds)) { if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) goto out_mds; pnfs_generic_pg_cleanup(pgio); /* Sleep for 1 second before retrying */ - ssleep(1); + if (!ff_layout_pg_retry_wait(pgio)) + return; goto retry; } - mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx); pgm = &pgio->pg_mirrors[0]; - pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize; + pgm->pg_bsize = mirror_ds->ds_versions[0].rsize; + nfs4_ff_layout_put_deviceid(mirror_ds); pgio->pg_mirror_idx = ds_idx; return; @@ -1076,19 +1088,34 @@ out_nolseg: if (pgio->pg_error < 0) { if (pgio->pg_error != -EAGAIN) return; - /* Retry getting layout segment if lower layer returned -EAGAIN */ - if (pgio->pg_maxretrans && req->wb_nio++ > pgio->pg_maxretrans) { - if (NFS_SERVER(pgio->pg_inode)->flags & NFS_MOUNT_SOFTERR) - pgio->pg_error = -ETIMEDOUT; - else - pgio->pg_error = -EIO; - return; - } - pgio->pg_error = 0; - /* Sleep for 1 second before retrying */ - ssleep(1); - goto retry; + goto retry_nolseg; } + /* + * No segment, and no error to report either: pnfs_update_layout() + * simply has nothing to give (NFS_LAYOUT_BULK_RECALL, a failed + * pnfs_layout_io_test, blocked LAYOUTGETs, a layout being + * returned). If the server forbids reading this file through the + * MDS there is no fallback to take, so wait for a layout on the + * same terms as the -EAGAIN above. The layout hdr that carried + * FF_FLAGS_NO_IO_THRU_MDS may itself be gone by now, which is why + * this asks the inode and not the hdr. + */ + if (!nfs_no_io_thru_mds(pgio->pg_inode)) + goto out_mds; +retry_nolseg: + /* Retry getting layout segment if lower layer returned -EAGAIN */ + if (pgio->pg_maxretrans && req->wb_nio++ > pgio->pg_maxretrans) { + if (NFS_SERVER(pgio->pg_inode)->flags & NFS_MOUNT_SOFTERR) + pgio->pg_error = -ETIMEDOUT; + else + pgio->pg_error = -EIO; + return; + } + pgio->pg_error = 0; + /* Sleep for 1 second before retrying */ + if (!ff_layout_pg_retry_wait(pgio)) + return; + goto retry; out_mds: trace_pnfs_mds_fallback_pg_init_read(pgio->pg_inode, 0, NFS4_MAX_UINT64, IOMODE_READ, @@ -1103,6 +1130,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) { struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds; struct nfs_pgio_mirror *pgm; struct nfs4_pnfs_ds *ds; u32 i, dss_id; @@ -1134,18 +1162,23 @@ retry: FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit, mirror->dss_count, req_offset(req)); + mirror_ds = ff_layout_get_mirror_ds(pgio->pg_lseg->pls_layout, + mirror, dss_id); ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, - dss_id, true); + mirror_ds, dss_id, OP_WRITE); if (IS_ERR(ds)) { + nfs4_ff_layout_put_deviceid(mirror_ds); if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) goto out_mds; pnfs_generic_pg_cleanup(pgio); /* Sleep for 1 second before retrying */ - ssleep(1); + if (!ff_layout_pg_retry_wait(pgio)) + return; goto retry; } pgm = &pgio->pg_mirrors[i]; - pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize; + pgm->pg_bsize = mirror_ds->ds_versions[0].wsize; + nfs4_ff_layout_put_deviceid(mirror_ds); } if (NFS_SERVER(pgio->pg_inode)->flags & @@ -1277,14 +1310,16 @@ static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr) u32 idx = hdr->pgio_mirror_idx + 1; u32 new_idx = 0; u32 dss_id = 0; - struct nfs4_pnfs_ds *ds; + struct nfs4_ff_layout_ds *mirror_ds; - ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx, - hdr->args.offset, &dss_id); - if (IS_ERR(ds)) - pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg); - else + mirror_ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx, + hdr->args.offset, &dss_id); + if (IS_ERR(mirror_ds)) { + pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg, NULL); + } else { + nfs4_ff_layout_put_deviceid(mirror_ds); ff_layout_send_layouterror(hdr->lseg); + } pnfs_read_resend_pnfs(hdr, new_idx); } @@ -1293,7 +1328,7 @@ static void ff_layout_reset_read(struct nfs_pgio_header *hdr) struct rpc_task *task = &hdr->task; pnfs_layoutcommit_inode(hdr->inode, false); - pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg); + pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg, NULL); if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { dprintk("%s Reset task %5u for i/o through MDS " @@ -1317,11 +1352,10 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, struct nfs4_state *state, struct nfs_client *clp, struct pnfs_layout_segment *lseg, - u32 idx, u32 dss_id) + struct nfs4_deviceid_node *devid) { struct pnfs_layout_hdr *lo = lseg->pls_layout; struct inode *inode = lo->plh_inode; - struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); struct nfs4_slot_table *tbl = nfs4_has_session(clp) ? &clp->cl_session->fc_slot_table : clp->cl_slot_tbl; @@ -1394,8 +1428,9 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, case -ENODEV: dprintk("%s DS connection error %d\n", __func__, task->tk_status); - nfs4_delete_deviceid(devid->ld, devid->nfs_client, - &devid->deviceid); + if (devid) + nfs4_delete_deviceid(devid->ld, devid->nfs_client, + &devid->deviceid); rpc_wake_up(&tbl->slot_tbl_waitq); break; default: @@ -1405,6 +1440,23 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, if (ff_layout_avoid_mds_available_ds(lseg)) return -NFS4ERR_RESET_TO_PNFS; reset: + /* + * FF_FLAGS_NO_IO_THRU_MDS: never resend through the MDS. The + * caller would do so with force_mds set (ff_layout_reset_read(), + * ff_layout_reset_write(hdr, false) -> pnfs_*_done_resend_to_mds()), + * which builds a descriptor out of the plain MDS page ops and so + * consults no layout at all -- this is the last point at which the + * policy can still be applied. Retry through pNFS instead, which + * takes a fresh LAYOUTGET; that is also the right answer for the + * invalid-layout cases that jump here, since they have just called + * pnfs_destroy_layout(). Having done so they can no longer ask the + * layout hdr about the flag, hence the inode. + */ + if (nfs_no_io_thru_mds(inode)) { + dprintk("%s Retry through pNFS, no MDS fallback. Error %d\n", + __func__, task->tk_status); + return -NFS4ERR_RESET_TO_PNFS; + } dprintk("%s Retry through MDS. Error %d\n", __func__, task->tk_status); return -NFS4ERR_RESET_TO_MDS; @@ -1419,9 +1471,8 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task, u32 op_status, struct nfs_client *clp, struct pnfs_layout_segment *lseg, - u32 idx, u32 dss_id) + struct nfs4_deviceid_node *devid) { - struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); switch (op_status) { case NFS_OK: @@ -1467,8 +1518,9 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task, default: dprintk("%s DS connection error %d\n", __func__, task->tk_status); - nfs4_delete_deviceid(devid->ld, devid->nfs_client, - &devid->deviceid); + if (devid) + nfs4_delete_deviceid(devid->ld, devid->nfs_client, + &devid->deviceid); } out_reset_to_pnfs: /* FIXME: Need to prevent infinite looping here. */ @@ -1485,12 +1537,13 @@ static int ff_layout_async_handle_error(struct rpc_task *task, struct nfs4_state *state, struct nfs_client *clp, struct pnfs_layout_segment *lseg, - u32 idx, u32 dss_id) + struct nfs4_deviceid_node *devid) { int vers = clp->cl_nfs_mod->rpc_vers->number; if (task->tk_status >= 0) { - ff_layout_mark_ds_reachable(lseg, idx, dss_id); + if (devid) + nfs4_mark_deviceid_available(devid); return 0; } @@ -1501,10 +1554,10 @@ static int ff_layout_async_handle_error(struct rpc_task *task, switch (vers) { case 3: return ff_layout_async_handle_error_v3(task, op_status, clp, - lseg, idx, dss_id); + lseg, devid); case 4: return ff_layout_async_handle_error_v4(task, op_status, state, - clp, lseg, idx, dss_id); + clp, lseg, devid); default: /* should never happen */ WARN_ON_ONCE(1); @@ -1513,6 +1566,7 @@ static int ff_layout_async_handle_error(struct rpc_task *task, } static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, + struct nfs4_deviceid_node *devid, u32 idx, u32 dss_id, u64 offset, u64 length, u32 *op_status, int opnum, int error) { @@ -1562,8 +1616,8 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, mirror = FF_LAYOUT_COMP(lseg, idx); err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), - mirror, dss_id, offset, length, status, opnum, - nfs_io_gfp_mask()); + mirror, devid, dss_id, offset, length, + status, opnum, nfs_io_gfp_mask()); /* * I/O we cancelled ourselves to return a recalled or revoked layout @@ -1580,7 +1634,8 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, case NFS4ERR_PERM: break; case NFS4ERR_NXIO: - ff_layout_mark_ds_unreachable(lseg, idx, dss_id); + if (devid) + nfs4_mark_deviceid_unavailable(devid); /* * Don't return the layout if this is a read and we still * have layouts to try @@ -1590,13 +1645,23 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, fallthrough; default: pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, - lseg); + lseg, + &mirror->dss[dss_id].devid); } out: dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status); } +static bool ff_layout_io_state_lost(const struct nfs_pgio_header *hdr) +{ + struct nfs_open_context *ctx = hdr->args.context; + + return test_bit(NFS_CONTEXT_BAD, &ctx->flags) || + !nfs4_valid_open_stateid(ctx->state) || + nfs4_lock_context_lost(ctx->state, hdr->args.lock_context); +} + /* NFS_PROTO call done callback routines */ static int ff_layout_read_done_cb(struct rpc_task *task, struct nfs_pgio_header *hdr) @@ -1608,8 +1673,14 @@ static int ff_layout_read_done_cb(struct rpc_task *task, hdr->args.offset); int err; + if (task->tk_status < 0 && !RPC_WAS_SENT(task) && + ff_layout_io_state_lost(hdr)) { + task->tk_status = -EIO; + return 0; + } + if (task->tk_status < 0) { - ff_layout_io_track_ds_error(hdr->lseg, + ff_layout_io_track_ds_error(hdr->lseg, hdr->ds_dev, hdr->pgio_mirror_idx, dss_id, hdr->args.offset, hdr->args.count, &hdr->res.op_status, OP_READ, @@ -1620,8 +1691,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task, err = ff_layout_async_handle_error(task, hdr->res.op_status, hdr->args.context->state, hdr->ds_clp, hdr->lseg, - hdr->pgio_mirror_idx, - dss_id); + hdr->ds_dev); trace_nfs4_pnfs_read(hdr, err); clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); @@ -1724,7 +1794,7 @@ static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, static int ff_layout_read_prepare_common(struct rpc_task *task, struct nfs_pgio_header *hdr) { - if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + if (unlikely(ff_layout_io_state_lost(hdr))) { rpc_exit(task, -EIO); return -EIO; } @@ -1813,8 +1883,14 @@ static int ff_layout_write_done_cb(struct rpc_task *task, loff_t end_offs = 0; int err; + if (task->tk_status < 0 && !RPC_WAS_SENT(task) && + ff_layout_io_state_lost(hdr)) { + task->tk_status = -EIO; + return 0; + } + if (task->tk_status < 0) { - ff_layout_io_track_ds_error(hdr->lseg, + ff_layout_io_track_ds_error(hdr->lseg, hdr->ds_dev, hdr->pgio_mirror_idx, dss_id, hdr->args.offset, hdr->args.count, &hdr->res.op_status, OP_WRITE, @@ -1825,8 +1901,7 @@ static int ff_layout_write_done_cb(struct rpc_task *task, err = ff_layout_async_handle_error(task, hdr->res.op_status, hdr->args.context->state, hdr->ds_clp, hdr->lseg, - hdr->pgio_mirror_idx, - dss_id); + hdr->ds_dev); trace_nfs4_pnfs_write(hdr, err); clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); @@ -1868,7 +1943,7 @@ static int ff_layout_commit_done_cb(struct rpc_task *task, u32 dss_id = calc_dss_id_from_commit(data->lseg, data->ds_commit_index); if (task->tk_status < 0) { - ff_layout_io_track_ds_error(data->lseg, idx, dss_id, + ff_layout_io_track_ds_error(data->lseg, data->ds_dev, idx, dss_id, data->args.offset, data->args.count, &data->res.op_status, OP_COMMIT, task->tk_status); @@ -1876,8 +1951,8 @@ static int ff_layout_commit_done_cb(struct rpc_task *task, } err = ff_layout_async_handle_error(task, data->res.op_status, - NULL, data->ds_clp, data->lseg, idx, - dss_id); + NULL, data->ds_clp, data->lseg, + data->ds_dev); trace_nfs4_pnfs_commit_ds(data, err); switch (err) { @@ -1950,7 +2025,7 @@ static void ff_layout_write_record_layoutstats_done(struct rpc_task *task, static int ff_layout_write_prepare_common(struct rpc_task *task, struct nfs_pgio_header *hdr) { - if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + if (unlikely(ff_layout_io_state_lost(hdr))) { rpc_exit(task, -EIO); return -EIO; } @@ -2167,6 +2242,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) struct rpc_clnt *ds_clnt; struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds; const struct cred *ds_cred; loff_t offset = hdr->args.offset; u32 idx = hdr->pgio_mirror_idx; @@ -2184,22 +2260,25 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) FF_LAYOUT_LSEG(lseg)->stripe_unit, mirror->dss_count, offset); - ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, false); + mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id, + OP_READ); if (IS_ERR(ds)) { ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; } - ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, - hdr->inode, dss_id); + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp, + hdr->inode); if (IS_ERR(ds_clnt)) goto out_failed; - ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id); + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, + mirror_ds, dss_id); if (!ds_cred) goto out_failed; - vers = nfs4_ff_layout_ds_version(mirror, dss_id); + vers = nfs4_ff_layout_ds_version(mirror_ds); dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), vers); @@ -2211,7 +2290,8 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) if (fh) hdr->args.fh = fh; - nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid); + nfs4_ff_layout_select_ds_stateid(mirror, mirror_ds, dss_id, + &hdr->args.stateid); /* * Note that if we ever decide to split across DSes, @@ -2228,6 +2308,10 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) ff_layout_read_record_layoutstats_start(&hdr->task, hdr); } + /* Transfer the device node reference to the I/O; put on release */ + pnfs_put_ds_dev(hdr->ds_dev); + hdr->ds_dev = &mirror_ds->id_node; + /* Perform an asynchronous read to ds */ nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_read_call_ops_v3 : @@ -2237,6 +2321,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) return PNFS_ATTEMPTED; out_failed: + nfs4_ff_layout_put_deviceid(mirror_ds); if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; if (ff_layout_no_fallback_to_mds(lseg)) { @@ -2244,7 +2329,8 @@ out_failed: * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, * never fall through to MDS I/O. */ - pnfs_error_mark_layout_for_return(hdr->inode, lseg); + pnfs_error_mark_layout_for_return(hdr->inode, lseg, + &mirror->dss[dss_id].devid); return PNFS_TRY_AGAIN; } trace_pnfs_mds_fallback_read_pagelist(hdr->inode, @@ -2262,6 +2348,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) struct rpc_clnt *ds_clnt; struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds; const struct cred *ds_cred; loff_t offset = hdr->args.offset; int vers; @@ -2275,22 +2362,25 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) FF_LAYOUT_LSEG(lseg)->stripe_unit, mirror->dss_count, offset); - ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true); + mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id, + OP_WRITE); if (IS_ERR(ds)) { ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; } - ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, - hdr->inode, dss_id); + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp, + hdr->inode); if (IS_ERR(ds_clnt)) goto out_failed; - ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id); + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, + mirror_ds, dss_id); if (!ds_cred) goto out_failed; - vers = nfs4_ff_layout_ds_version(mirror, dss_id); + vers = nfs4_ff_layout_ds_version(mirror_ds); dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n", __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count, @@ -2305,7 +2395,8 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) if (fh) hdr->args.fh = fh; - nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid); + nfs4_ff_layout_select_ds_stateid(mirror, mirror_ds, dss_id, + &hdr->args.stateid); /* * Note that if we ever decide to split across DSes, @@ -2321,6 +2412,10 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) ff_layout_write_record_layoutstats_start(&hdr->task, hdr); } + /* Transfer the device node reference to the I/O; put on release */ + pnfs_put_ds_dev(hdr->ds_dev); + hdr->ds_dev = &mirror_ds->id_node; + /* Perform an asynchronous write */ nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_write_call_ops_v3 : @@ -2330,6 +2425,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) return PNFS_ATTEMPTED; out_failed: + nfs4_ff_layout_put_deviceid(mirror_ds); if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; if (ff_layout_no_fallback_to_mds(lseg)) { @@ -2337,7 +2433,8 @@ out_failed: * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, * never fall through to MDS I/O. */ - pnfs_error_mark_layout_for_return(hdr->inode, lseg); + pnfs_error_mark_layout_for_return(hdr->inode, lseg, + &mirror->dss[dss_id].devid); return PNFS_TRY_AGAIN; } trace_pnfs_mds_fallback_write_pagelist(hdr->inode, @@ -2364,6 +2461,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) struct rpc_clnt *ds_clnt; struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds = NULL; const struct cred *ds_cred; u32 idx, dss_id; int vers, ret; @@ -2376,20 +2474,23 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) idx = calc_mirror_idx_from_commit(lseg, data->ds_commit_index); mirror = FF_LAYOUT_COMP(lseg, idx); dss_id = calc_dss_id_from_commit(lseg, data->ds_commit_index); - ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true); + mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id, + OP_COMMIT); if (IS_ERR(ds)) goto out_err; - ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, - data->inode, dss_id); + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp, + data->inode); if (IS_ERR(ds_clnt)) goto out_err; - ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred, dss_id); + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred, + mirror_ds, dss_id); if (!ds_cred) goto out_err; - vers = nfs4_ff_layout_ds_version(mirror, dss_id); + vers = nfs4_ff_layout_ds_version(mirror_ds); dprintk("%s ino %llu, how %d cl_count %d vers %d\n", __func__, data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count), @@ -2410,6 +2511,10 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) ff_layout_commit_record_layoutstats_start(&data->task, data); } + /* Transfer the device node reference to the commit; put on release */ + pnfs_put_ds_dev(data->ds_dev); + data->ds_dev = &mirror_ds->id_node; + ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_commit_call_ops_v3 : &ff_layout_commit_call_ops_v4, @@ -2417,6 +2522,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) put_cred(ds_cred); return ret; out_err: + nfs4_ff_layout_put_deviceid(mirror_ds); pnfs_generic_prepare_to_resend_writes(data); pnfs_generic_commit_release(data); return -EAGAIN; @@ -2459,7 +2565,8 @@ static bool ff_layout_match_io(const struct rpc_task *task, const void *data) return false; } -static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg) +static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg, + const struct nfs4_deviceid *devid) { struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); struct nfs4_ff_layout_mirror *mirror; @@ -2472,22 +2579,94 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg) for (idx = 0; idx < flseg->mirror_array_cnt; idx++) { mirror = flseg->mirror_array[idx]; for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { - mirror_ds = mirror->dss[dss_id].mirror_ds; - if (IS_ERR_OR_NULL(mirror_ds)) + if (devid && memcmp(&mirror->dss[dss_id].devid, devid, + sizeof(*devid)) != 0) continue; - ds = mirror->dss[dss_id].mirror_ds->ds; + rcu_read_lock(); + mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds); + if (IS_ERR_OR_NULL(mirror_ds) || + !atomic_inc_not_zero(&mirror_ds->id_node.ref)) { + rcu_read_unlock(); + continue; + } + rcu_read_unlock(); + ds = mirror_ds->ds; if (!ds) - continue; + goto next; ds_clp = ds->ds_clp; if (!ds_clp) - continue; + goto next; clnt = ds_clp->cl_rpcclient; if (!clnt) - continue; + goto next; if (!rpc_cancel_tasks(clnt, -ECANCELED, ff_layout_match_io, lseg)) - continue; + goto next; rpc_clnt_disconnect(clnt); +next: + nfs4_ff_layout_put_deviceid(mirror_ds); + } + } +} + +/* Called under @lo's inode i_lock. */ +static bool ff_layout_references_deviceid(struct pnfs_layout_hdr *lo, + const struct nfs4_deviceid *id) +{ + struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo); + struct nfs4_ff_layout_mirror *mirror; + u32 dss_id; + + list_for_each_entry(mirror, &flo->mirrors, mirrors) + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) + if (memcmp(&mirror->dss[dss_id].devid, id, + sizeof(*id)) == 0) + return true; + return false; +} + +/* + * Un-pin every stripe node resolved from @id: in-flight I/O drains on the + * old node through its own reference, the next I/O re-resolves. + */ +static void ff_layout_reresolve_deviceid(struct pnfs_layout_hdr *lo, + const struct nfs4_deviceid *id, + bool immediate, + struct list_head *head) +{ + struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo); + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *old; + struct nfs4_deviceid_put *put; + u32 dss_id; + + list_for_each_entry(mirror, &flo->mirrors, mirrors) { + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { + if (memcmp(&mirror->dss[dss_id].devid, id, + sizeof(*id)) != 0) + continue; + /* Allocate before un-pinning: on failure the reference + * stays put rather than being dropped here, where the + * final put may not sleep. + */ + put = kzalloc_obj(*put, GFP_ATOMIC); + if (!put) + continue; + old = unrcu_pointer( + xchg(&mirror->dss[dss_id].mirror_ds, NULL)); + if (IS_ERR_OR_NULL(old)) { + kfree(put); + continue; + } + /* A node still hashed was fetched after the unhash + * and carries the new mapping; mark only the + * superseded ones. + */ + if (immediate && + hlist_unhashed_lockless(&old->id_node.node)) + nfs4_mark_deviceid_unavailable(&old->id_node); + put->dev = &old->id_node; + list_add(&put->node, head); } } } @@ -2700,7 +2879,7 @@ ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args) spin_lock(&args->inode->i_lock); ff_args->num_dev = ff_layout_mirror_prepare_stats( - &ff_layout->generic_hdr, &ff_args->devinfo[0], + &ff_layout->generic_hdr, &ff_args->devinfo[0], &ff_args->priv[0], ARRAY_SIZE(ff_args->devinfo), NFS4_FF_OP_LAYOUTRETURN); spin_unlock(&args->inode->i_lock); @@ -2875,10 +3054,11 @@ ff_layout_encode_io_latency(struct xdr_stream *xdr, static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr, const struct nfs42_layoutstat_devinfo *devinfo, - struct nfs4_ff_layout_ds_stripe *dss_info) + struct nfs4_ff_layoutstat_priv *priv) { + struct nfs4_ff_layout_ds_stripe *dss_info = priv->dss_info; struct nfs4_pnfs_ds_addr *da; - struct nfs4_pnfs_ds *ds = dss_info->mirror_ds->ds; + struct nfs4_pnfs_ds *ds = priv->mirror_ds->ds; struct nfs_fh *fh = &dss_info->fh_versions[0]; __be32 *p; @@ -2925,10 +3105,10 @@ ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args, static void ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque) { - struct nfs4_ff_layout_ds_stripe *dss_info = opaque->data; - struct nfs4_ff_layout_mirror *mirror = dss_info->mirror; + struct nfs4_ff_layoutstat_priv *priv = opaque->data; - ff_layout_put_mirror(mirror); + nfs4_ff_layout_put_deviceid(priv->mirror_ds); + ff_layout_put_mirror(priv->dss_info->mirror); } static const struct nfs4_xdr_opaque_ops layoutstat_ops = { @@ -2939,20 +3119,23 @@ static const struct nfs4_xdr_opaque_ops layoutstat_ops = { static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, struct nfs42_layoutstat_devinfo *devinfo, + struct nfs4_ff_layoutstat_priv *priv, int dev_limit, enum nfs4_ff_op_type type) { struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); struct nfs4_ff_layout_mirror *mirror; struct nfs4_ff_layout_ds_stripe *dss_info; - struct nfs4_deviceid_node *dev; + struct nfs4_ff_layout_ds *mirror_ds; int i = 0, dss_id; + rcu_read_lock(); list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) { for (dss_id = 0; dss_id < mirror->dss_count; ++dss_id) { dss_info = &mirror->dss[dss_id]; if (i >= dev_limit) break; - if (IS_ERR_OR_NULL(dss_info->mirror_ds)) + mirror_ds = rcu_dereference(dss_info->mirror_ds); + if (IS_ERR_OR_NULL(mirror_ds)) continue; if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags) && @@ -2961,9 +3144,12 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, /* mirror refcount put in cleanup_layoutstats */ if (!refcount_inc_not_zero(&mirror->ref)) continue; - dev = &dss_info->mirror_ds->id_node; + /* The pin holds a reference; it is exchanged out only + * under i_lock. Put in ff_layout_free_layoutstats(). + */ + atomic_inc(&mirror_ds->id_node.ref); memcpy(&devinfo->dev_id, - &dev->deviceid, + &mirror_ds->id_node.deviceid, NFS4_DEVICEID4_SIZE); devinfo->offset = 0; devinfo->length = NFS4_MAX_UINT64; @@ -2979,12 +3165,16 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, spin_unlock(&mirror->lock); devinfo->layout_type = LAYOUT_FLEX_FILES; devinfo->ld_private.ops = &layoutstat_ops; - devinfo->ld_private.data = &mirror->dss[dss_id]; + priv->dss_info = dss_info; + priv->mirror_ds = mirror_ds; + devinfo->ld_private.data = priv; devinfo++; + priv++; i++; } } + rcu_read_unlock(); return i; } @@ -2992,21 +3182,28 @@ static int ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args) { struct pnfs_layout_hdr *lo; struct nfs4_flexfile_layout *ff_layout; + struct nfs4_ff_layoutstat_priv *priv; const int dev_count = PNFS_LAYOUTSTATS_MAXDEV; - /* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */ - args->devinfo = kmalloc_objs(*args->devinfo, dev_count, - nfs_io_gfp_mask()); + /* + * For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics. + * The per-devinfo private entries are co-allocated after the + * devinfo array and freed along with it. + */ + args->devinfo = kmalloc(dev_count * (sizeof(*args->devinfo) + + sizeof(*priv)), + nfs_io_gfp_mask()); if (!args->devinfo) return -ENOMEM; + priv = (struct nfs4_ff_layoutstat_priv *)&args->devinfo[dev_count]; spin_lock(&args->inode->i_lock); lo = NFS_I(args->inode)->layout; if (lo && pnfs_layout_is_valid(lo)) { ff_layout = FF_LAYOUT_FROM_HDR(lo); args->num_dev = ff_layout_mirror_prepare_stats( - &ff_layout->generic_hdr, &args->devinfo[0], dev_count, - NFS4_FF_OP_LAYOUTSTATS); + &ff_layout->generic_hdr, &args->devinfo[0], priv, + dev_count, NFS4_FF_OP_LAYOUTSTATS); } else args->num_dev = 0; spin_unlock(&args->inode->i_lock); @@ -3055,6 +3252,8 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = { .pg_write_ops = &ff_layout_pg_write_ops, .get_ds_info = ff_layout_get_ds_info, .free_deviceid_node = ff_layout_free_deviceid_node, + .reresolve_deviceid = ff_layout_reresolve_deviceid, + .layout_references_deviceid = ff_layout_references_deviceid, .read_pagelist = ff_layout_read_pagelist, .write_pagelist = ff_layout_write_pagelist, .alloc_deviceid_node = ff_layout_alloc_deviceid_node, diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h index a5bd00f69e82..09c3cd6964fd 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.h +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -79,7 +79,7 @@ struct nfs4_ff_layout_ds_stripe { struct nfs4_ff_layout_mirror *mirror; struct nfs4_deviceid devid; u32 efficiency; - struct nfs4_ff_layout_ds *mirror_ds; + struct nfs4_ff_layout_ds __rcu *mirror_ds; u32 fh_versions_cnt; struct nfs_fh *fh_versions; nfs4_stateid stateid; @@ -124,9 +124,20 @@ struct nfs4_flexfile_layout { unsigned long flags; }; +/* + * Per-devinfo private data for a layoutstats/layoutreturn encode: the + * stripe the stats describe plus a reference on its device node so the + * node (and its DS addresses) stay valid until the XDR encode runs. + */ +struct nfs4_ff_layoutstat_priv { + struct nfs4_ff_layout_ds_stripe *dss_info; + struct nfs4_ff_layout_ds *mirror_ds; +}; + struct nfs4_flexfile_layoutreturn_args { struct list_head errors; struct nfs42_layoutstat_devinfo devinfo[FF_LAYOUTSTATS_MAXDEV]; + struct nfs4_ff_layoutstat_priv priv[FF_LAYOUTSTATS_MAXDEV]; unsigned int num_errors; unsigned int num_dev; struct page *pages[1]; @@ -162,20 +173,6 @@ FF_LAYOUT_COMP(struct pnfs_layout_segment *lseg, u32 idx) return NULL; } -static inline struct nfs4_deviceid_node * -FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) -{ - struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, idx); - - if (mirror != NULL) { - struct nfs4_ff_layout_ds *mirror_ds = mirror->dss[dss_id].mirror_ds; - - if (!IS_ERR_OR_NULL(mirror_ds)) - return &mirror_ds->id_node; - } - return NULL; -} - static inline u32 FF_LAYOUT_MIRROR_COUNT(struct pnfs_layout_segment *lseg) { @@ -207,9 +204,9 @@ ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg) } static inline int -nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_mirror *mirror, u32 dss_id) +nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_ds *mirror_ds) { - return mirror->dss[dss_id].mirror_ds->ds_versions[0].version; + return mirror_ds->ds_versions[0].version; } static inline u32 @@ -220,7 +217,7 @@ nfs4_ff_layout_calc_dss_id(const u64 stripe_unit, const u32 dss_count, const lof if (dss_count == 1 || stripe_unit == 0) return 0; - do_div(tmp, stripe_unit); + tmp = div64_u64(tmp, stripe_unit); return do_div(tmp, dss_count); } @@ -232,6 +229,7 @@ void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds); void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds); int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, struct nfs4_ff_layout_mirror *mirror, + const struct nfs4_deviceid_node *devid, u32 dss_id, u64 offset, u64 length, int status, enum nfs_opnum4 opnum, gfp_t gfp_flags); void ff_layout_send_layouterror(struct pnfs_layout_segment *lseg); @@ -245,23 +243,29 @@ struct nfs_fh * nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id); void nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror, + const struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id, nfs4_stateid *stateid); +struct nfs4_ff_layout_ds * +ff_layout_get_mirror_ds(struct pnfs_layout_hdr *lo, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id); struct nfs4_pnfs_ds * nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, struct nfs4_ff_layout_mirror *mirror, + struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id, - bool fail_return); + enum nfs_opnum4 opnum); struct rpc_clnt * -nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror, +nfs4_ff_find_or_create_ds_client(const struct nfs4_ff_layout_ds *mirror_ds, struct nfs_client *ds_clp, - struct inode *inode, - u32 dss_id); + struct inode *inode); const struct cred *ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror, const struct pnfs_layout_range *range, const struct cred *mdscred, + const struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id); bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg); bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg); diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index 5c0216bd5fce..3af0e6d361a5 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -20,6 +20,7 @@ static unsigned int dataserver_timeo = NFS_DEF_TCP_TIMEO; static unsigned int dataserver_retrans; +static unsigned int dataserver_nconnect; static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg); @@ -159,26 +160,13 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, goto out_err_drain_dsaddrs; /* If DS was already in cache, free ds addrs */ - while (!list_empty(&dsaddrs)) { - da = list_first_entry(&dsaddrs, - struct nfs4_pnfs_ds_addr, - da_node); - list_del_init(&da->da_node); - kfree(da->da_remotestr); - kfree(da); - } + nfs4_pnfs_ds_addr_list_free(&dsaddrs); folio_put(scratch); return new_ds; out_err_drain_dsaddrs: - while (!list_empty(&dsaddrs)) { - da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr, - da_node); - list_del_init(&da->da_node); - kfree(da->da_remotestr); - kfree(da); - } + nfs4_pnfs_ds_addr_list_free(&dsaddrs); kfree(ds_versions); out_scratch: @@ -256,6 +244,7 @@ ff_layout_add_ds_error_locked(struct nfs4_flexfile_layout *flo, int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, struct nfs4_ff_layout_mirror *mirror, + const struct nfs4_deviceid_node *devid, u32 dss_id, u64 offset, u64 length, int status, enum nfs_opnum4 opnum, gfp_t gfp_flags) { @@ -264,7 +253,7 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, if (status == 0) return 0; - if (IS_ERR_OR_NULL(mirror->dss[dss_id].mirror_ds)) + if (devid == NULL) return -EINVAL; dserr = kmalloc_obj(*dserr, gfp_flags); @@ -277,8 +266,7 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, dserr->status = status; dserr->opnum = opnum; nfs4_stateid_copy(&dserr->stateid, &mirror->dss[dss_id].stateid); - memcpy(&dserr->deviceid, &mirror->dss[dss_id].mirror_ds->id_node.deviceid, - NFS4_DEVICEID4_SIZE); + memcpy(&dserr->deviceid, &devid->deviceid, NFS4_DEVICEID4_SIZE); spin_lock(&flo->generic_hdr.plh_inode->i_lock); ff_layout_add_ds_error_locked(flo, dserr); @@ -317,50 +305,81 @@ nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id) void nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror, + const struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id, nfs4_stateid *stateid) { - if (nfs4_ff_layout_ds_version(mirror, dss_id) == 4) + if (nfs4_ff_layout_ds_version(mirror_ds) == 4) nfs4_stateid_copy(stateid, &mirror->dss[dss_id].stateid); } -static bool -ff_layout_init_mirror_ds(struct pnfs_layout_hdr *lo, - struct nfs4_ff_layout_mirror *mirror, - u32 dss_id) +/* + * Resolve the stripe's deviceid on first use and pin the node on the + * mirror. Returns a node the caller must put, or an ERR_PTR. + */ +struct nfs4_ff_layout_ds * +ff_layout_get_mirror_ds(struct pnfs_layout_hdr *lo, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id) { + struct nfs4_ff_layout_ds *mirror_ds, *old; + struct nfs4_deviceid_node *node; + if (mirror == NULL) - goto outerr; - if (mirror->dss[dss_id].mirror_ds == NULL) { - struct nfs4_deviceid_node *node; - struct nfs4_ff_layout_ds *mirror_ds = ERR_PTR(-ENODEV); + return ERR_PTR(-ENODEV); - node = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), - &mirror->dss[dss_id].devid, lo->plh_lc_cred, - GFP_KERNEL); - if (node) - mirror_ds = FF_LAYOUT_MIRROR_DS(node); +retry: + rcu_read_lock(); + mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds); + if (mirror_ds && !IS_ERR(mirror_ds) && + atomic_inc_not_zero(&mirror_ds->id_node.ref)) { + rcu_read_unlock(); + return mirror_ds; + } + rcu_read_unlock(); + if (IS_ERR(mirror_ds)) + return mirror_ds; + if (mirror_ds != NULL) + /* raced with a reset; the field is being re-pointed */ + goto retry; - /* check for race with another call to this function */ - if (cmpxchg(&mirror->dss[dss_id].mirror_ds, NULL, mirror_ds) && - mirror_ds != ERR_PTR(-ENODEV)) - nfs4_put_deviceid_node(node); + node = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), + &mirror->dss[dss_id].devid, lo->plh_lc_cred, + GFP_KERNEL); + if (node) { + mirror_ds = FF_LAYOUT_MIRROR_DS(node); + /* + * Take the caller's reference before the pointer becomes + * visible below, so a concurrent reset of the installed + * pointer cannot drop the last reference under us. + */ + atomic_inc(&node->ref); + } else { + mirror_ds = ERR_PTR(-ENODEV); } - if (IS_ERR(mirror->dss[dss_id].mirror_ds)) - goto outerr; + /* check for race with another call to this function */ + old = unrcu_pointer(cmpxchg(&mirror->dss[dss_id].mirror_ds, + NULL, RCU_INITIALIZER(mirror_ds))); + if (old == NULL) + return mirror_ds; - return true; -outerr: - return false; + /* lost the race; use the winner's node instead */ + if (node) { + nfs4_put_deviceid_node(node); + nfs4_put_deviceid_node(node); + } + goto retry; } /** * nfs4_ff_layout_prepare_ds - prepare a DS connection for an RPC call * @lseg: the layout segment we're operating on * @mirror: layout mirror describing the DS to use + * @mirror_ds: referenced device node for the stripe, from + * ff_layout_get_mirror_ds() (may be an ERR_PTR) * @dss_id: DS stripe id to select stripe to use - * @fail_return: return layout on connect failure? + * @opnum: operation this connection is being prepared for * * Try to prepare a DS connection to accept an RPC call. This involves * selecting a mirror to use and connecting the client to it if it's not @@ -368,16 +387,19 @@ outerr: * * Since we only need a single functioning mirror to satisfy a read, we don't * want to return the layout if there is one. For writes though, any down - * mirror should result in a LAYOUTRETURN. @fail_return is how we distinguish - * between the two cases. + * mirror should result in a LAYOUTRETURN. @opnum is how we distinguish + * between the two cases. On failure, @opnum is also reported in the tracked + * device error so that the server can tell which class of I/O the client + * was unable to send to the mirror. * - * Returns a pointer to a connected DS object on success or NULL on failure. + * Returns a pointer to a connected DS object on success or an ERR_PTR value on failure. */ struct nfs4_pnfs_ds * nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, struct nfs4_ff_layout_mirror *mirror, + struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id, - bool fail_return) + enum nfs_opnum4 opnum) { struct nfs4_pnfs_ds *ds; struct inode *ino = lseg->pls_layout->plh_inode; @@ -385,10 +407,10 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, unsigned int max_payload; int status = -EAGAIN; - if (!ff_layout_init_mirror_ds(lseg->pls_layout, mirror, dss_id)) + if (IS_ERR_OR_NULL(mirror_ds)) goto noconnect; - ds = mirror->dss[dss_id].mirror_ds->ds; + ds = mirror_ds->ds; if (READ_ONCE(ds->ds_clp)) goto out; /* matching smp_wmb() in _nfs4_pnfs_v3/4_ds_connect */ @@ -397,11 +419,12 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, /* FIXME: For now we assume the server sent only one version of NFS * to use for the DS. */ - status = nfs4_pnfs_ds_connect(s, ds, &mirror->dss[dss_id].mirror_ds->id_node, + status = nfs4_pnfs_ds_connect(s, ds, &mirror_ds->id_node, dataserver_timeo, dataserver_retrans, - mirror->dss[dss_id].mirror_ds->ds_versions[0].version, - mirror->dss[dss_id].mirror_ds->ds_versions[0].minor_version, - mirror->dss[dss_id].mirror_ds->ds_versions[0].tightly_coupled); + dataserver_nconnect, + mirror_ds->ds_versions[0].version, + mirror_ds->ds_versions[0].minor_version, + mirror_ds->ds_versions[0].tightly_coupled); /* connect success, check rsize/wsize limit */ if (!status) { @@ -414,20 +437,29 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, max_payload = nfs_block_size(rpc_max_payload(ds->ds_clp->cl_rpcclient), NULL); - if (mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize > max_payload) - mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize = max_payload; - if (mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize > max_payload) - mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize = max_payload; + if (mirror_ds->ds_versions[0].rsize > max_payload) + mirror_ds->ds_versions[0].rsize = max_payload; + if (mirror_ds->ds_versions[0].wsize > max_payload) + mirror_ds->ds_versions[0].wsize = max_payload; + goto out; + } + /* The connect was interrupted by a signal: this is not a DS error */ + if (status == -ERESTARTSYS || status == -EINTR) { + ds = ERR_PTR(status); goto out; } noconnect: ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), - mirror, dss_id, lseg->pls_range.offset, + mirror, + IS_ERR_OR_NULL(mirror_ds) ? + NULL : &mirror_ds->id_node, + dss_id, lseg->pls_range.offset, lseg->pls_range.length, NFS4ERR_NXIO, - OP_ILLEGAL, GFP_NOIO); + opnum, GFP_NOIO); ff_layout_send_layouterror(lseg); - if (fail_return || !ff_layout_has_available_ds(lseg)) - pnfs_error_mark_layout_for_return(ino, lseg); + if (opnum != OP_READ || !ff_layout_has_available_ds(lseg)) + pnfs_error_mark_layout_for_return(ino, lseg, + &mirror->dss[dss_id].devid); ds = ERR_PTR(status); out: return ds; @@ -437,11 +469,12 @@ const struct cred * ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror, const struct pnfs_layout_range *range, const struct cred *mdscred, + const struct nfs4_ff_layout_ds *mirror_ds, u32 dss_id) { const struct cred *cred; - if (mirror && !mirror->dss[dss_id].mirror_ds->ds_versions[0].tightly_coupled) { + if (mirror && !mirror_ds->ds_versions[0].tightly_coupled) { cred = ff_layout_get_mirror_cred(mirror, range->iomode, dss_id); if (!cred) cred = get_cred(mdscred); @@ -453,20 +486,18 @@ ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror, /** * nfs4_ff_find_or_create_ds_client - Find or create a DS rpc client - * @mirror: pointer to the mirror + * @mirror_ds: device node for the stripe * @ds_clp: nfs_client for the DS * @inode: pointer to inode - * @dss_id: DS stripe id * * Find or create a DS rpc client with th MDS server rpc client auth flavor * in the nfs_client cl_ds_clients list. */ struct rpc_clnt * -nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror, - struct nfs_client *ds_clp, struct inode *inode, - u32 dss_id) +nfs4_ff_find_or_create_ds_client(const struct nfs4_ff_layout_ds *mirror_ds, + struct nfs_client *ds_clp, struct inode *inode) { - switch (mirror->dss[dss_id].mirror_ds->ds_versions[0].version) { + switch (mirror_ds->ds_versions[0].version) { case 3: /* For NFSv3 DS, flavor is set when creating DS connections */ return ds_clp->cl_rpcclient; @@ -571,49 +602,60 @@ unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo, static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg) { struct nfs4_ff_layout_mirror *mirror; - struct nfs4_deviceid_node *devid; + struct nfs4_ff_layout_ds *mirror_ds; + bool ret = false; u32 idx, dss_id; + rcu_read_lock(); for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) { mirror = FF_LAYOUT_COMP(lseg, idx); if (!mirror) continue; for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { - if (!mirror->dss[dss_id].mirror_ds) - return true; - if (IS_ERR(mirror->dss[dss_id].mirror_ds)) + mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds); + if (!mirror_ds) { + ret = true; + goto out; + } + if (IS_ERR(mirror_ds)) continue; - devid = &mirror->dss[dss_id].mirror_ds->id_node; - if (!nfs4_test_deviceid_unavailable(devid)) - return true; + if (!nfs4_test_deviceid_unavailable(&mirror_ds->id_node)) { + ret = true; + goto out; + } } } - - return false; +out: + rcu_read_unlock(); + return ret; } static bool ff_rw_layout_has_available_ds(struct pnfs_layout_segment *lseg) { struct nfs4_ff_layout_mirror *mirror; - struct nfs4_deviceid_node *devid; + struct nfs4_ff_layout_ds *mirror_ds; + bool ret = false; u32 idx, dss_id; + rcu_read_lock(); for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) { mirror = FF_LAYOUT_COMP(lseg, idx); if (!mirror) - return false; + goto out; for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { - if (IS_ERR(mirror->dss[dss_id].mirror_ds)) - return false; - if (!mirror->dss[dss_id].mirror_ds) + mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds); + if (IS_ERR(mirror_ds)) + goto out; + if (!mirror_ds) continue; - devid = &mirror->dss[dss_id].mirror_ds->id_node; - if (nfs4_test_deviceid_unavailable(devid)) - return false; + if (nfs4_test_deviceid_unavailable(&mirror_ds->id_node)) + goto out; } } - - return FF_LAYOUT_MIRROR_COUNT(lseg) != 0; + ret = FF_LAYOUT_MIRROR_COUNT(lseg) != 0; +out: + rcu_read_unlock(); + return ret; } static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg) @@ -644,3 +686,8 @@ module_param(dataserver_timeo, uint, 0644); MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the " "NFSv4.1 client waits for a response from a " " data server before it retries an NFS request."); +module_param(dataserver_nconnect, uint, 0644); +MODULE_PARM_DESC(dataserver_nconnect, "The maximum number of connections " + "the NFSv4.1 client opens to each data server, " + "capping the value inherited from the MDS nconnect " + "mount option. 0 (default) applies no cap."); diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 1967de7d1dff..0f65cd4aa591 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -402,6 +402,7 @@ static void nfs_set_mount_transport_protocol(struct nfs_fs_context *ctx) ctx->mount_server.protocol = XPRT_TRANSPORT_UDP; break; case XPRT_TRANSPORT_TCP: + case XPRT_TRANSPORT_TCP_TLS: case XPRT_TRANSPORT_RDMA: ctx->mount_server.protocol = XPRT_TRANSPORT_TCP; } @@ -1750,6 +1751,7 @@ static int nfs_init_fs_context(struct fs_context *fc) ctx->minorversion = 0; ctx->need_mount = true; ctx->xprtsec.policy = RPC_XPRTSEC_NONE; + ctx->xprtsec.keyring_serial = TLS_NO_KEYRING; ctx->xprtsec.cert_serial = TLS_NO_CERT; ctx->xprtsec.privkey_serial = TLS_NO_PRIVKEY; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 3c9b2ec4e244..e0a2e92952af 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -42,6 +42,12 @@ #include #include #include +#include +#include +#include +#ifdef CONFIG_KEYS +#include +#endif #include "nfs4_fs.h" #include "callback.h" @@ -2641,11 +2647,131 @@ static int nfsiod_start(void) unsigned int nfs_net_id; EXPORT_SYMBOL_GPL(nfs_net_id); +#ifdef CONFIG_KEYS +static int nfs_init_keyring(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + struct cred *cred; + struct key *keyring; + + cred = prepare_kernel_cred(&init_task); + if (!cred) + return -ENOMEM; + keyring = keyring_alloc(".nfs", net->user_ns->owner, + net->user_ns->group, cred, + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + (KEY_USR_ALL & ~KEY_USR_SETATTR), + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + put_cred(cred); + if (IS_ERR(keyring)) + return PTR_ERR(keyring); + nn->nfs_keyring = keyring; + return 0; +} + +static void nfs_exit_keyring(struct nfs_net *nn) +{ + key_put(nn->nfs_keyring); +} + +static int nfs_keyring_vet_description(const char *desc) +{ + return strcmp(desc, ".nfs") ? -EINVAL : 0; +} + +static int nfs_keyring_format_serial(char *buf, size_t len) +{ + struct key *keyring = nfs_net_keyring(current->nsproxy->net_ns); + + return snprintf(buf, len, "%d", key_serial(keyring)); +} + +/* + * A key planted by add_key() answers request_key() before the handler + * runs. Accept only the serial number the handler would produce. + */ +static int nfs_keyring_preparse(struct key_preparsed_payload *prep) +{ + char serial[12]; + int len; + + len = nfs_keyring_format_serial(serial, sizeof(serial)); + if (prep->datalen != len || !prep->data || + memcmp(prep->data, serial, len)) + return -EINVAL; + return user_preparse(prep); +} + +static int nfs_keyring_request_key(struct key *authkey, void *aux) +{ + struct request_key_auth *rka = get_request_key_auth(authkey); + char serial[12]; + int len, ret; + + len = nfs_keyring_format_serial(serial, sizeof(serial)); + ret = key_instantiate_and_link(rka->target_key, serial, len, + rka->dest_keyring, authkey); + if (ret < 0) + complete_request_key(authkey, ret); + return ret; +} + +/* + * A session keyring survives setns(). Without the net domain tag, a + * key instantiated in one namespace answers a request from another. + */ +static struct key_type key_type_nfs_keyring = { + .name = "nfs_keyring", + .flags = KEY_TYPE_NET_DOMAIN, + .vet_description = nfs_keyring_vet_description, + .preparse = nfs_keyring_preparse, + .free_preparse = user_free_preparse, + .instantiate = generic_key_instantiate, + .revoke = user_revoke, + .destroy = user_destroy, + .describe = user_describe, + .read = user_read, + .request_key = nfs_keyring_request_key, +}; + +static int __init nfs_register_key_type(void) +{ + return register_key_type(&key_type_nfs_keyring); +} + +static void nfs_unregister_key_type(void) +{ + unregister_key_type(&key_type_nfs_keyring); +} +#else +static inline int nfs_init_keyring(struct net *net) +{ + return 0; +} + +static inline void nfs_exit_keyring(struct nfs_net *nn) +{ +} + +static inline int nfs_register_key_type(void) +{ + return 0; +} + +static inline void nfs_unregister_key_type(void) +{ +} +#endif /* CONFIG_KEYS */ + static int nfs_net_init(struct net *net) { struct nfs_net *nn = net_generic(net, nfs_net_id); int err; + err = nfs_init_keyring(net); + if (err) + return err; + nfs_clients_init(net); if (!rpc_proc_register(net, &nn->rpcstats)) { @@ -2663,14 +2789,18 @@ err_proc_nfs: rpc_proc_unregister(net, "nfs"); err_proc_rpc: nfs_clients_exit(net); + nfs_exit_keyring(nn); return err; } static void nfs_net_exit(struct net *net) { + struct nfs_net *nn = net_generic(net, nfs_net_id); + rpc_proc_unregister(net, "nfs"); nfs_fs_proc_net_exit(net); nfs_clients_exit(net); + nfs_exit_keyring(nn); } static struct pernet_operations nfs_net_ops = { @@ -2680,35 +2810,6 @@ static struct pernet_operations nfs_net_ops = { .size = sizeof(struct nfs_net), }; -#ifdef CONFIG_KEYS -static struct key *nfs_keyring; - -static int __init nfs_init_keyring(void) -{ - nfs_keyring = keyring_alloc(".nfs", - GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, - current_cred(), - (KEY_POS_ALL & ~KEY_POS_SETATTR) | - (KEY_USR_ALL & ~KEY_USR_SETATTR), - KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); - return PTR_ERR_OR_ZERO(nfs_keyring); -} - -static void nfs_exit_keyring(void) -{ - key_put(nfs_keyring); -} -#else -static inline int nfs_init_keyring(void) -{ - return 0; -} - -static inline void nfs_exit_keyring(void) -{ -} -#endif /* CONFIG_KEYS */ - /* * Initialize NFS */ @@ -2716,71 +2817,71 @@ static int __init init_nfs_fs(void) { int err; - err = nfs_init_keyring(); - if (err) - return err; - err = nfs_sysfs_init(); if (err < 0) - goto out10; + return err; err = register_pernet_subsys(&nfs_net_ops); if (err < 0) - goto out9; + goto err_sysfs; + + err = nfs_register_key_type(); + if (err) + goto err_pernet; err = nfsiod_start(); if (err) - goto out7; + goto err_keytype; err = nfs_fs_proc_init(); if (err) - goto out6; + goto err_nfsiod; err = nfs_init_nfspagecache(); if (err) - goto out5; + goto err_proc; err = nfs_init_inodecache(); if (err) - goto out4; + goto err_nfspagecache; err = nfs_init_readpagecache(); if (err) - goto out3; + goto err_inodecache; err = nfs_init_writepagecache(); if (err) - goto out2; + goto err_readpagecache; err = nfs_init_directcache(); if (err) - goto out1; + goto err_writepagecache; err = register_nfs_fs(); if (err) - goto out0; + goto err_directcache; return 0; -out0: +err_directcache: nfs_destroy_directcache(); -out1: +err_writepagecache: nfs_destroy_writepagecache(); -out2: +err_readpagecache: nfs_destroy_readpagecache(); -out3: +err_inodecache: nfs_destroy_inodecache(); -out4: +err_nfspagecache: nfs_destroy_nfspagecache(); -out5: +err_proc: nfs_fs_proc_exit(); -out6: +err_nfsiod: nfsiod_stop(); -out7: +err_keytype: + nfs_unregister_key_type(); +err_pernet: unregister_pernet_subsys(&nfs_net_ops); -out9: +err_sysfs: nfs_sysfs_exit(); -out10: - nfs_exit_keyring(); return err; } @@ -2791,12 +2892,12 @@ static void __exit exit_nfs_fs(void) nfs_destroy_readpagecache(); nfs_destroy_inodecache(); nfs_destroy_nfspagecache(); + nfs_unregister_key_type(); unregister_pernet_subsys(&nfs_net_ops); unregister_nfs_fs(); nfs_fs_proc_exit(); nfsiod_stop(); nfs_sysfs_exit(); - nfs_exit_keyring(); } /* Not quite true; I just maintain it */ diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index cbb323c6530d..5af6a032f145 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -86,6 +86,7 @@ struct nfs_client_initdata { struct xprtsec_parms xprtsec; unsigned long connect_timeout; unsigned long reconnect_timeout; + rpc_authflavor_t authflavor; /* Auth flavor to use for DS connection */ }; /* @@ -251,6 +252,7 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, + unsigned int ds_nconnect, u32 minor_version, bool tightly_coupled); extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *, @@ -260,7 +262,7 @@ extern void nfs4_session_limit_xasize(struct nfs_server *server); extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, - unsigned int ds_retrans); + unsigned int ds_retrans, unsigned int ds_nconnect); #ifdef CONFIG_PROC_FS extern int __init nfs_fs_proc_init(void); extern void nfs_fs_proc_exit(void); @@ -394,6 +396,7 @@ extern unsigned long nfs_access_cache_count(struct shrinker *shrink, struct shrink_control *sc); extern unsigned long nfs_access_cache_scan(struct shrinker *shrink, struct shrink_control *sc); +extern void nfs_drop_nlink(struct inode *inode, unsigned long gencount); struct dentry *nfs_lookup(struct inode *, struct dentry *, unsigned int); void nfs_d_prune_case_insensitive_aliases(struct inode *inode); int nfs_create(const struct mnt_idmap *, struct inode *, struct dentry *, @@ -545,6 +548,35 @@ static inline bool nfs_file_io_is_buffered(struct nfs_inode *nfsi) return test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0; } +/* + * Module-private nfs_inode->flags bit (not in , so no + * exported layout changes): set, and never cleared for the life of the + * in-core inode, once a layout driver has seen the server forbid I/O + * through the MDS for this file (flexfiles sets it from + * FF_FLAGS_NO_IO_THRU_MDS in ff_layout_alloc_lseg()). The RFC 5661 + * mdsthreshold hint must not be acted on for such a file, since the only + * thing pnfs_within_mdsthreshold() can ask for is the one thing the + * layout forbids. Servers are assumed to be consistent in their + * no-fallback policy per file, the same assumption + * ff_layout_hdr_no_fallback_to_mds() already makes; if one were not, the + * only effect is that its mdsthreshold hint - a SHOULD - stops being + * honored for an inode that is already in core. + */ +#define NFS_INO_NO_IO_THRU_MDS (30) + +static inline void nfs_set_no_io_thru_mds(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + if (!test_bit(NFS_INO_NO_IO_THRU_MDS, &nfsi->flags)) + set_bit(NFS_INO_NO_IO_THRU_MDS, &nfsi->flags); +} + +static inline bool nfs_no_io_thru_mds(struct inode *inode) +{ + return test_bit(NFS_INO_NO_IO_THRU_MDS, &NFS_I(inode)->flags); +} + /* Must be called with exclusively locked inode->i_rwsem */ static inline void nfs_file_block_o_direct(struct nfs_inode *nfsi) { diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index f42b6112a613..cdada6cbb13b 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -446,7 +446,7 @@ nfs_local_iters_setup_dio(struct nfs_local_kiocb *iocb, int rw, if (unlikely(!iocb->iter_is_dio_aligned[n_iters])) { trace_nfs_local_dio_misaligned(iocb->hdr->inode, - local_dio->start_len, local_dio->middle_len, local_dio); + local_dio->middle_offset, local_dio->middle_len, local_dio); return 0; /* no DIO-aligned IO possible */ } iocb->end_iter_index = n_iters; @@ -674,6 +674,8 @@ static void nfs_local_call_read(struct work_struct *work) n_iters = atomic_read(&iocb->n_iters); for (int i = 0; i < n_iters ; i++) { + size_t expected; + if (iocb->iter_is_dio_aligned[i]) { iocb->kiocb.ki_flags |= IOCB_DIRECT; /* Only use AIO completion if DIO-aligned segment is last */ @@ -684,6 +686,8 @@ static void nfs_local_call_read(struct work_struct *work) } else iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + /* read_iter() advances the iterator: measure it beforehand */ + expected = iov_iter_count(&iocb->iters[i]); scoped_with_creds(filp->f_cred) status = filp->f_op->read_iter(&iocb->kiocb, &iocb->iters[i]); @@ -691,7 +695,7 @@ static void nfs_local_call_read(struct work_struct *work) continue; /* Break on completion, errors, or short reads */ if (nfs_local_pgio_done(iocb, status) || status < 0 || - (size_t)status < iov_iter_count(&iocb->iters[i])) { + (size_t)status < expected) { nfs_local_read_iocb_done(iocb); break; } @@ -890,7 +894,7 @@ static void nfs_local_call_write(struct work_struct *work) file_start_write(filp); n_iters = atomic_read(&iocb->n_iters); for (int i = 0; i < n_iters ; i++) { - size_t icount; + size_t expected; if (iocb->iter_is_dio_aligned[i]) { iocb->kiocb.ki_flags |= IOCB_DIRECT; @@ -902,16 +906,17 @@ static void nfs_local_call_write(struct work_struct *work) } else iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + /* write_iter() advances the iterator: measure it beforehand */ + expected = iov_iter_count(&iocb->iters[i]); scoped_with_creds(filp->f_cred) status = filp->f_op->write_iter(&iocb->kiocb, &iocb->iters[i]); if (status == -EIOCBQUEUED) continue; /* Break on completion, errors, or short writes */ - icount = iov_iter_count(&iocb->iters[i]); if (nfs_local_pgio_done(iocb, status) || status < 0 || - (size_t)status < icount) { - if ((size_t)status < icount) { + (size_t)status < expected) { + if ((size_t)status < expected) { struct nfs_lock_context *ctx = iocb->hdr->req->wb_lock_context; @@ -931,6 +936,7 @@ static void nfs_local_do_write(struct nfs_local_kiocb *iocb, const struct rpc_call_ops *call_ops) { struct nfs_pgio_header *hdr = iocb->hdr; + enum nfs3_stable_how committed = hdr->args.stable; dprintk("%s: vfs_write count=%u pos=%llu %s\n", __func__, hdr->args.count, hdr->args.offset, @@ -949,9 +955,20 @@ static void nfs_local_do_write(struct nfs_local_kiocb *iocb, iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC; } + /* + * Report the stability the write will actually have. A DIO WRITE + * is persisted before it completes whatever was asked for, see + * nfs_local_iters_init(), and a caller told so has no reason to + * COMMIT data that is already on stable storage. + */ + if (iocb->kiocb.ki_flags & IOCB_SYNC) + committed = NFS_FILE_SYNC; + else if (iocb->kiocb.ki_flags & IOCB_DSYNC) + committed = NFS_DATA_SYNC; + nfs_local_pgio_init(hdr, call_ops); - nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable); + nfs_set_local_verifier(hdr->inode, hdr->res.verf, committed); INIT_WORK(&iocb->work, nfs_local_call_write); if (nfs_local_defer_io()) diff --git a/fs/nfs/netns.h b/fs/nfs/netns.h index e1decff366d4..de2d0df29060 100644 --- a/fs/nfs/netns.h +++ b/fs/nfs/netns.h @@ -18,6 +18,7 @@ struct bl_dev_msg { uint32_t major, minor; }; +struct key; struct nfs_netns_client; struct nfs_net { @@ -34,11 +35,15 @@ struct nfs_net { unsigned short nfs_callback_tcpport; unsigned short nfs_callback_tcpport6; int cb_users[NFS4_MAX_MINOR_VERSION + 1]; - struct list_head nfs4_data_server_cache; +#define NFS4_DS_CACHE_HASH_BITS 8 +#define NFS4_DS_CACHE_HASH_SIZE (1 << NFS4_DS_CACHE_HASH_BITS) + /* every entry is still in bucket 0 until the key is added */ + struct hlist_head nfs4_data_server_cache[NFS4_DS_CACHE_HASH_SIZE]; spinlock_t nfs4_data_server_lock; #endif /* CONFIG_NFS_V4 */ struct nfs_netns_client *nfs_client; spinlock_t nfs_client_lock; + struct key *nfs_keyring; ktime_t boot_time; struct rpc_stat rpcstats; #ifdef CONFIG_PROC_FS @@ -48,4 +53,11 @@ struct nfs_net { extern unsigned int nfs_net_id; +static inline struct key *nfs_net_keyring(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + return nn->nfs_keyring; +} + #endif diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c index 5d97c1d38bb6..4d9d43f687e3 100644 --- a/fs/nfs/nfs3client.c +++ b/fs/nfs/nfs3client.c @@ -84,7 +84,8 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source, */ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, const struct sockaddr_storage *ds_addr, int ds_addrlen, - int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans) + int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, + unsigned int ds_nconnect) { struct rpc_timeout ds_timeout; unsigned long connect_timeout = ds_timeo * (ds_retrans + 1) * HZ / 10; @@ -101,6 +102,7 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, .cred = mds_srv->cred, .xprtsec = { .policy = RPC_XPRTSEC_NONE, + .keyring_serial = TLS_NO_KEYRING, .cert_serial = TLS_NO_CERT, .privkey_serial = TLS_NO_PRIVKEY, }, @@ -124,8 +126,12 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, fallthrough; case XPRT_TRANSPORT_RDMA: case XPRT_TRANSPORT_TCP: - if (mds_clp->cl_nconnect > 1) + if (mds_clp->cl_nconnect > 1) { cl_init.nconnect = mds_clp->cl_nconnect; + if (ds_nconnect) + cl_init.nconnect = min(cl_init.nconnect, + ds_nconnect); + } } if (mds_srv->flags & NFS_MOUNT_NORESVPORT) diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index b80d0c5efc27..b5aa3100b627 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -448,13 +448,21 @@ nfs3_proc_remove(struct inode *dir, struct dentry *dentry) .rpc_resp = &res, }; int status = -ENOMEM; + struct inode *inode = d_inode(dentry); + unsigned long gencount; dprintk("NFS call remove %pd2\n", dentry); res.dir_attr = nfs_alloc_fattr(); if (res.dir_attr == NULL) goto out; + if (inode) + gencount = READ_ONCE(NFS_I(inode)->attr_gencount); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + if (status == 0 && inode) + nfs_drop_nlink(inode, gencount); + nfs_post_op_update_inode(dir, res.dir_attr); nfs_free_fattr(res.dir_attr); out: diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index b48e5b87cb2a..2ad2efc43bb9 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -52,6 +52,7 @@ enum nfs4_client_state { NFS4CLNT_RECALL_ANY_LAYOUT_READ, NFS4CLNT_RECALL_ANY_LAYOUT_RW, NFS4CLNT_DELEGRETURN_DELAYED, + NFS4CLNT_DEVICEID_DELETE, }; #define NFS4_RENEW_TIMEOUT 0x01 @@ -493,6 +494,7 @@ int nfs41_discover_server_trunking(struct nfs_client *clp, struct nfs_client **, const struct cred *); extern void nfs4_schedule_session_recovery(struct nfs4_session *, int); extern void nfs41_notify_server(struct nfs_client *); +extern void nfs4_deviceid_delete_recover_run(struct nfs_client *clp); bool nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1, struct nfs41_server_owner *o2); @@ -509,6 +511,7 @@ extern void nfs_inode_find_state_and_recover(struct inode *inode, const nfs4_stateid *stateid); extern int nfs4_state_mark_reclaim_nograce(struct nfs_client *, struct nfs4_state *); extern void nfs4_schedule_lease_recovery(struct nfs_client *); +extern void nfs4_reset_all_state(struct nfs_client *); extern int nfs4_wait_clnt_recover(struct nfs_client *clp); extern int nfs4_client_recover_expired_lease(struct nfs_client *clp); extern void nfs4_schedule_state_manager(struct nfs_client *); @@ -526,6 +529,8 @@ extern int nfs4_select_rw_stateid(struct nfs4_state *, fmode_t, const struct cred **); extern bool nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state); +extern bool nfs4_lock_context_lost(struct nfs4_state *state, + const struct nfs_lock_context *l_ctx); extern struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_mask); extern int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task); diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index b661f446ea49..440f8496b717 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -217,6 +217,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_last_renewal = jiffies; init_waitqueue_head(&clp->cl_lock_waitq); INIT_LIST_HEAD(&clp->pending_cb_stateids); + INIT_LIST_HEAD(&clp->cl_deviceid_deletes); if (cl_init->minorversion != 0) __set_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags); @@ -229,8 +230,12 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) /* * Set up the connection to the server before we add add to the * global list. + * + * Prefer krb5i unless the caller already knows AUTH_UNIX is required. */ - err = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_GSS_KRB5I); + err = -EINVAL; + if (cl_init->authflavor != RPC_AUTH_UNIX) + err = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_GSS_KRB5I); if (err == -EINVAL) err = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_UNIX); if (err < 0) @@ -286,6 +291,7 @@ static void nfs4_shutdown_client(struct nfs_client *clp) nfs4_kill_renewd(clp); clp->cl_mvops->shutdown_client(clp); nfs4_destroy_callback(clp); + pnfs_deviceid_delete_queue_free(clp); if (__test_and_clear_bit(NFS_CS_IDMAP, &clp->cl_res_state)) nfs_idmap_delete(clp); @@ -792,7 +798,8 @@ static int nfs4_set_client(struct nfs_server *server, struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, - u32 minor_version, bool tightly_coupled) + unsigned int ds_nconnect, u32 minor_version, + bool tightly_coupled) { struct rpc_timeout ds_timeout; struct nfs_client *mds_clp = mds_srv->nfs_client; @@ -809,9 +816,11 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, .cred = mds_srv->cred, .xprtsec = { .policy = RPC_XPRTSEC_NONE, + .keyring_serial = TLS_NO_KEYRING, .cert_serial = TLS_NO_CERT, .privkey_serial = TLS_NO_PRIVKEY, }, + .authflavor = mds_clp->cl_rpcclient->cl_auth->au_flavor, }; char buf[INET6_ADDRSTRLEN + 1]; @@ -830,6 +839,9 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, case XPRT_TRANSPORT_TCP: if (mds_clp->cl_nconnect > 1) { cl_init.nconnect = mds_clp->cl_nconnect; + if (ds_nconnect) + cl_init.nconnect = min(cl_init.nconnect, + ds_nconnect); cl_init.max_connect = NFS_MAX_TRANSPORTS; } } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 21e5df7e5251..e1018b03c1cb 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -4969,12 +4969,14 @@ static int nfs4_proc_remove(struct inode *dir, struct dentry *dentry) }; struct inode *inode = d_inode(dentry); int err; + unsigned long gencount; if (inode) { if (inode->i_nlink == 1) nfs4_inode_return_delegation(inode); else nfs4_inode_make_writeable(inode); + gencount = READ_ONCE(NFS_I(inode)->attr_gencount); } do { err = _nfs4_proc_remove(dir, &dentry->d_name, NF4REG); @@ -4982,6 +4984,9 @@ static int nfs4_proc_remove(struct inode *dir, struct dentry *dentry) err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); } while (exception.retry); + + if (err == 0 && inode) + nfs_drop_nlink(inode, gencount); return err; } @@ -9680,6 +9685,14 @@ nfs4_layoutget_handle_exception(struct rpc_task *task, case -NFS4ERR_BADLAYOUT: status = -EOVERFLOW; goto out; + /* + * NFS4ERR_TOOSMALL means the layout for the requested range + * exceeds what the client advertised in loga_maxcount (see + * RFC8881 section 18.43.3). + */ + case -ETOOSMALL: + status = -EMSGSIZE; + goto out; /* * NFS4ERR_LAYOUTTRYLATER is a conflict with another client * (or clients) writing to the same RAID stripe except when @@ -10498,6 +10511,134 @@ out_put_clp: return ret; } +/* + * GETDEVICEINFO surfacing the raw status; nfs4_get_device_info() + * swallows it. A device too large for one page fails with something + * other than -ENOENT, which still proves existence. + */ +static int nfs4_deviceid_validate(struct nfs_server *server, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *id, const struct cred *cred) +{ + struct pnfs_device pdev; + struct page *page; + int status; + + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + memset(&pdev, 0, sizeof(pdev)); + memcpy(&pdev.dev_id, id, sizeof(pdev.dev_id)); + pdev.layout_type = ld->id; + pdev.pages = &page; + pdev.pglen = PAGE_SIZE; + pdev.maxcount = PAGE_SIZE - nfs41_maxgetdevinfo_overhead; + + status = nfs4_proc_getdeviceinfo(server, &pdev, cred); + __free_page(page); + return status; +} + +/* + * A DELETE for a deviceID we still hold layouts on implies the server + * revoked them: run the RFC 8881 Section 18.40.4 recovery. A layout the + * server still calls valid leaves the revocations unable to confirm the + * delete, so verify it with GETDEVICEINFO. + */ +static void nfs4_deviceid_delete_recover(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *id) +{ + LIST_HEAD(layouts); + struct nfs4_deviceid_ref *ref, *confirm = NULL; + bool revoked = false; + bool inconclusive = false; + int status; + + if (pnfs_layout_collect_deviceid_refs(clp, ld, id, &layouts)) { + /* Only a partial list -- an allocation failed, or an inode is + * being evicted. Leave the device cached and recover on a + * later notification. + */ + pnfs_layout_put_deviceid_refs(&layouts); + return; + } + + if (list_empty(&layouts)) { + nfs4_delete_deviceid(ld, clp, id); + return; + } + + list_for_each_entry(ref, &layouts, node) { + struct pnfs_layout_hdr *lo = ref->lo; + struct inode *inode = ref->inode; + bool invalidated = false; + LIST_HEAD(head); + + status = nfs41_test_stateid(NFS_SERVER(inode), &ref->stateid, + ref->cred); + switch (status) { + case NFS_OK: + case -NFS4ERR_OLD_STATEID: + if (!confirm) + confirm = ref; + break; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + spin_lock(&inode->i_lock); + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&ref->stateid, + &lo->plh_stateid)) { + pnfs_mark_layout_stateid_invalid(lo, &head); + revoked = true; + invalidated = true; + } + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&head); + if (invalidated) + nfs_commit_inode(inode, 0); + nfs41_free_stateid(NFS_SERVER(inode), &ref->stateid, + ref->cred, true); + break; + default: + inconclusive = true; + break; + } + } + + if (confirm) { + status = nfs4_deviceid_validate(NFS_SERVER(confirm->inode), + ld, id, confirm->cred); + if (status == -ENOENT) { + /* Section 18.40.4 prescribes EXCHANGE_ID here; + * nfs4_schedule_lease_recovery() would only renew + * the existing lease. + */ + pr_warn_ratelimited("NFS: server %s deleted a deviceID referred to by a layout it still considers valid; re-establishing the client ID\n", + clp->cl_hostname); + nfs4_reset_all_state(clp); + nfs4_delete_deviceid(ld, clp, id); + } + } else if (revoked && !inconclusive) { + nfs4_delete_deviceid(ld, clp, id); + } + pnfs_layout_put_deviceid_refs(&layouts); +} + +void nfs4_deviceid_delete_recover_run(struct nfs_client *clp) +{ + struct nfs4_deviceid_delete *dd; + + while ((dd = pnfs_deviceid_delete_dequeue(clp)) != NULL) { + nfs4_deviceid_delete_recover(clp, dd->ld, &dd->id); + pnfs_put_layoutdriver(dd->ld); + kfree(dd); + } +} + static void nfs41_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) { diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index a5dec0473e22..8cd178f1da4c 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -812,10 +812,13 @@ void nfs4_close_sync(struct nfs4_state *state, fmode_t fmode) * be made. Failing that, a "consistent" choice is preferable. The * consistent choice we make is to prefer the first owner, that of a * Posix lock. + * + * Called with state->state_lock held. No reference is taken: a lock + * state is freed only after it is unlinked under state->state_lock. */ static struct nfs4_lock_state * -__nfs4_find_lock_state(struct nfs4_state *state, - fl_owner_t owner, fl_owner_t owner2) +__nfs4_lookup_lock_state(struct nfs4_state *state, + fl_owner_t owner, fl_owner_t owner2) { struct nfs4_lock_state *pos, *ret = NULL; list_for_each_entry(pos, &state->lock_states, ls_locks) { @@ -826,6 +829,16 @@ __nfs4_find_lock_state(struct nfs4_state *state, if (pos->ls_owner == owner2) ret = pos; } + return ret; +} + +static struct nfs4_lock_state * +__nfs4_find_lock_state(struct nfs4_state *state, + fl_owner_t owner, fl_owner_t owner2) +{ + struct nfs4_lock_state *ret; + + ret = __nfs4_lookup_lock_state(state, owner, owner2); if (ret) refcount_inc(&ret->ls_count); return ret; @@ -968,7 +981,7 @@ static int nfs4_copy_lock_stateid(nfs4_stateid *dst, fl_flock_owner = l_ctx->open_context->flock_owner; spin_lock(&state->state_lock); - lsp = __nfs4_find_lock_state(state, owner, fl_flock_owner); + lsp = __nfs4_lookup_lock_state(state, owner, fl_flock_owner); if (lsp && test_bit(NFS_LOCK_LOST, &lsp->ls_flags)) ret = -EIO; else if (lsp != NULL && test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) != 0) { @@ -976,11 +989,28 @@ static int nfs4_copy_lock_stateid(nfs4_stateid *dst, ret = 0; } spin_unlock(&state->state_lock); - nfs4_put_lock_state(lsp); out: return ret; } +bool nfs4_lock_context_lost(struct nfs4_state *state, + const struct nfs_lock_context *l_ctx) +{ + struct nfs4_lock_state *lsp; + bool ret; + + if (!test_bit(LK_STATE_IN_USE, &state->flags)) + return false; + + spin_lock(&state->state_lock); + lsp = __nfs4_lookup_lock_state(state, l_ctx->lockowner, + l_ctx->open_context->flock_owner); + ret = lsp && test_bit(NFS_LOCK_LOST, &lsp->ls_flags); + spin_unlock(&state->state_lock); + return ret; +} +EXPORT_SYMBOL_GPL(nfs4_lock_context_lost); + bool nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state) { bool ret; @@ -2354,7 +2384,7 @@ void nfs41_notify_server(struct nfs_client *clp) nfs4_schedule_state_manager(clp); } -static void nfs4_reset_all_state(struct nfs_client *clp) +void nfs4_reset_all_state(struct nfs_client *clp) { if (test_and_set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state) == 0) { set_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state); @@ -2669,6 +2699,11 @@ static void nfs4_state_manager(struct nfs_client *clp) set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); } nfs4_layoutreturn_any_run(clp); + if (test_and_clear_bit(NFS4CLNT_DEVICEID_DELETE, + &clp->cl_state)) { + nfs4_deviceid_delete_recover_run(clp); + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + } clear_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state); } diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index fc049ce4ba8a..740f3bc4a19d 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -5169,8 +5169,7 @@ static int decode_lock_denied(struct xdr_stream *xdr, struct file_lock *fl) p = xdr_decode_hyper(p, &clientid); /* read 8 bytes */ namelen = be32_to_cpup(p); /* read 4 bytes */ /* have read all 32 bytes now */ p = xdr_inline_decode(xdr, namelen); /* variable size field */ - if (likely(!p)) - return -EIO; + /* We have an owner here, return DENIED */ return -NFS4ERR_DENIED; } @@ -6187,7 +6186,7 @@ static int decode_layoutget(struct xdr_stream *xdr, struct rpc_rqst *req, dprintk("NFS: server cheating in layoutget reply: " "layout len %u > recvd %u\n", res->layoutp->len, recvd); - status = -EINVAL; + status = -EMSGSIZE; goto out; } diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index b15c1732c869..a32c76df72db 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -1772,7 +1772,10 @@ DECLARE_EVENT_CLASS(nfs_local_dio_class, __entry->count = count; __entry->mem_align = local_dio->mem_align; __entry->offset_align = local_dio->offset_align; - __entry->start = offset; + if (local_dio->start_len) + __entry->start = local_dio->middle_offset - local_dio->start_len; + else + __entry->start = 0; __entry->start_len = local_dio->start_len; __entry->middle = local_dio->middle_offset; __entry->middle_len = local_dio->middle_len; diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 7dd478ffc2fa..71f0ce2bc4ea 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -404,20 +404,28 @@ static struct nfs_page *nfs_page_create(struct nfs_lock_context *l_ctx, return req; } -static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio) +static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio, + bool pinned) { if (folio != NULL) { req->wb_folio = folio; - folio_get(folio); + if (pinned) + set_bit(PG_PINNED, &req->wb_flags); + else + folio_get(folio); set_bit(PG_FOLIO, &req->wb_flags); } } -static void nfs_page_assign_page(struct nfs_page *req, struct page *page) +static void nfs_page_assign_page(struct nfs_page *req, struct page *page, + bool pinned) { if (page != NULL) { req->wb_page = page; - get_page(page); + if (pinned) + set_bit(PG_PINNED, &req->wb_flags); + else + get_page(page); } } @@ -425,6 +433,7 @@ static void nfs_page_assign_page(struct nfs_page *req, struct page *page) * nfs_page_create_from_page - Create an NFS read/write request. * @ctx: open context to use * @page: page to write + * @pinned: true if page is pinned * @pgbase: starting offset within the page for the write * @offset: file offset for the write * @count: number of bytes to read/write @@ -435,6 +444,7 @@ static void nfs_page_assign_page(struct nfs_page *req, struct page *page) */ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, struct page *page, + bool pinned, unsigned int pgbase, loff_t offset, unsigned int count) { @@ -446,7 +456,9 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, ret = nfs_page_create(l_ctx, pgbase, offset >> PAGE_SHIFT, offset_in_page(offset), count); if (!IS_ERR(ret)) { - nfs_page_assign_page(ret, page); + nfs_page_assign_page(ret, page, pinned); + if (pinned) + ret->wb_nr_pinned = 1; nfs_page_group_init(ret, NULL); } nfs_put_lock_context(l_ctx); @@ -457,6 +469,7 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, * nfs_page_create_from_folio - Create an NFS read/write request. * @ctx: open context to use * @folio: folio to write + * @pinned: true if folio is pinned * @offset: starting offset within the folio for the write * @count: number of bytes to read/write * @@ -466,6 +479,7 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, */ struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, struct folio *folio, + bool pinned, unsigned int offset, unsigned int count) { @@ -476,7 +490,10 @@ struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, return ERR_CAST(l_ctx); ret = nfs_page_create(l_ctx, offset, folio->index, offset, count); if (!IS_ERR(ret)) { - nfs_page_assign_folio(ret, folio); + nfs_page_assign_folio(ret, folio, pinned); + if (pinned) + ret->wb_nr_pinned = nfs_page_array_len(offset_in_page(offset), + count); nfs_page_group_init(ret, NULL); } nfs_put_lock_context(l_ctx); @@ -498,9 +515,11 @@ nfs_create_subreq(struct nfs_page *req, offset, count); if (!IS_ERR(ret)) { if (folio) - nfs_page_assign_folio(ret, folio); + nfs_page_assign_folio(ret, folio, + test_bit(PG_PINNED, &req->wb_flags)); else - nfs_page_assign_page(ret, page); + nfs_page_assign_page(ret, page, + test_bit(PG_PINNED, &req->wb_flags)); /* find the last request */ for (last = req->wb_head; last->wb_this_page != req->wb_head; @@ -552,11 +571,21 @@ static void nfs_clear_request(struct nfs_page *req) struct nfs_open_context *ctx; if (folio != NULL) { - folio_put(folio); + if (test_and_clear_bit(PG_PINNED, &req->wb_flags)) { + if (req->wb_nr_pinned > 0) + unpin_user_folio(folio, req->wb_nr_pinned); + } else { + folio_put(folio); + } req->wb_folio = NULL; clear_bit(PG_FOLIO, &req->wb_flags); } else if (page != NULL) { - put_page(page); + if (test_and_clear_bit(PG_PINNED, &req->wb_flags)) { + if (req->wb_nr_pinned > 0) + unpin_user_pages(&page, req->wb_nr_pinned); + } else { + put_page(page); + } req->wb_page = NULL; } if (l_ctx != NULL) { @@ -599,6 +628,23 @@ void nfs_release_request(struct nfs_page *req) } EXPORT_SYMBOL_GPL(nfs_release_request); +/* + * nfs_release_request_list - Release a list of NFS read/write requests + * @head: list of requests to release + * + * Removes each request from the list and drops it's refcount. + */ +void nfs_release_request_list(struct list_head *head) +{ + while (!list_empty(head)) { + struct nfs_page *req = nfs_list_entry(head->next); + + nfs_list_remove_request(req); + nfs_release_request(req); + } +} +EXPORT_SYMBOL_GPL(nfs_release_request_list); + /* * nfs_generic_pg_test - determine if requests can be coalesced * @desc: pointer to descriptor diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 4f9c0f639014..a057a06318c7 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -433,7 +433,7 @@ bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, } /* Try to update the seqid to the most recent */ err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, - true); + true, NULL); if (err != -EBUSY) { dst->seqid = lo->plh_stateid.seqid; *dst_range = range; @@ -487,7 +487,8 @@ static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, .length = NFS4_MAX_UINT64, }; - return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq, true); + return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq, true, + NULL); } static int @@ -525,7 +526,7 @@ pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) spin_lock(&inode->i_lock); pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); - pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, true); + pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, true, NULL); spin_unlock(&inode->i_lock); pnfs_free_lseg_list(&head); dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, @@ -740,7 +741,7 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, if (mark_lseg_invalid(lseg, tmp_list)) continue; remaining++; - pnfs_lseg_cancel_io(server, lseg); + pnfs_lseg_cancel_io(server, lseg, NULL); } dprintk("%s:Return %i\n", __func__, remaining); return remaining; @@ -1167,11 +1168,12 @@ pnfs_alloc_init_layoutget_args(struct inode *ino, struct nfs_open_context *ctx, const nfs4_stateid *stateid, const struct pnfs_layout_range *range, - gfp_t gfp_flags) + size_t min_reply_sz, gfp_t gfp_flags) { struct nfs_server *server = pnfs_find_server(ino, ctx); size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; - size_t max_pages = max_response_pages(server); + size_t session_pages = max_response_pages(server); + size_t max_pages = session_pages; struct nfs4_layoutget *lgp; dprintk("--> %s\n", __func__); @@ -1186,6 +1188,19 @@ pnfs_alloc_init_layoutget_args(struct inode *ino, max_pages = npages; } + /* + * A previous LAYOUTGET on this layout or on this server did not + * fit the reply buffer: raise the layout driver's default up to + * the session's maximum response size. + */ + if (!min_reply_sz) + min_reply_sz = READ_ONCE(server->lg_reply_sz); + if (min_reply_sz) { + size_t npages = (min_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (npages > max_pages) + max_pages = min(npages, session_pages); + } + lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); if (!lgp->args.layout.pages) { kfree(lgp); @@ -1210,7 +1225,7 @@ pnfs_alloc_init_layoutget_args(struct inode *ino, lgp->args.minlength = i_size - range->offset; } } - lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; + lgp->args.maxcount = lgp->args.layout.pglen; pnfs_copy_range(&lgp->args.range, range); lgp->args.type = server->pnfs_curr_ld->id; lgp->args.inode = ino; @@ -1462,7 +1477,7 @@ _pnfs_return_layout(struct inode *ino) } valid_layout = pnfs_layout_is_valid(lo); pnfs_clear_layoutcommit(ino, &tmp_list); - pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0, true); + pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0, true, NULL); /* Don't send a LAYOUTRETURN if list was initially empty */ @@ -2004,7 +2019,9 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo, * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a * WRITE request. * - * A return of true means use MDS I/O. + * A return of true means use MDS I/O, so for a file whose layout forbids + * that (see NFS_INO_NO_IO_THRU_MDS) the hint is not evaluated at all: + * there is no answer it could give that this client may act on. * * From rfc 5661: * If a file's size is smaller than the file size threshold, data accesses @@ -2025,6 +2042,14 @@ static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, if (t == NULL) return ret; + /* + * The server has told a layout driver that this file's I/O may not + * go through the MDS (flexfiles FF_FLAGS_NO_IO_THRU_MDS). Honor + * that over its own mdsthreshold hint. + */ + if (nfs_no_io_thru_mds(ino)) + return ret; + dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); @@ -2145,6 +2170,7 @@ pnfs_update_layout(struct inode *ino, .inode = ino, }; unsigned long giveup = jiffies + (clp->cl_lease_time << 1); + size_t reply_sz = 0; bool first; if (!pnfs_enabled_sb(NFS_SERVER(ino))) { @@ -2304,7 +2330,8 @@ lookup_again: if (arg.length != NFS4_MAX_UINT64) arg.length = PAGE_ALIGN(arg.length); - lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, reply_sz, + gfp_flags); if (!lgp) { lseg = ERR_PTR(-ENOMEM); trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, @@ -2335,6 +2362,25 @@ lookup_again: lo, pnfs_iomode_to_fail_bit(iomode)); lseg = NULL; goto out_put_layout_hdr; + case -EMSGSIZE: { + /* + * The layout exceeded loga_maxcount (NFS4ERR_TOOSMALL): + * retry once with the reply buffer raised to the + * session's maximum response size before falling back + * to I/O through the MDS. + */ + size_t max = max_response_pages(server) << PAGE_SHIFT; + + if (reply_sz < max) { + reply_sz = max; + exception.retry = 1; + break; + } + pnfs_layout_set_fail_bit( + lo, pnfs_iomode_to_fail_bit(iomode)); + lseg = NULL; + goto out_put_layout_hdr; + } default: if (!nfs_error_is_fatal(PTR_ERR(lseg))) { pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); @@ -2354,6 +2400,13 @@ lookup_again: goto lookup_again; } } else { + /* + * A LAYOUTGET that only succeeded with an escalated reply + * buffer: remember the size so that future LAYOUTGETs to + * this server skip the attempt at the driver's default. + */ + if (reply_sz) + WRITE_ONCE(server->lg_reply_sz, reply_sz); pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); } @@ -2448,7 +2501,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data, lo = _pnfs_grab_empty_layout(ino, ctx); if (!lo) return; - lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 0, nfs_io_gfp_mask()); if (!lgp) { clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); @@ -2474,7 +2527,7 @@ static void _lgopen_prepare_floating(struct nfs4_opendata *data, }; struct nfs4_layoutget *lgp; - lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, 0, nfs_io_gfp_mask()); if (!lgp) return; @@ -2616,7 +2669,8 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) .iomode = IOMODE_ANY, .length = NFS4_MAX_UINT64, }; - pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0, true); + pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0, true, + NULL); goto out_forget; } else { /* We have a completely new layout */ @@ -2649,6 +2703,7 @@ out_forget: * @return_range: describe layout segment ranges to be returned * @seq: stateid seqid to match * @cancel_io: signal io be cancelled + * @devid: only cancel io directed at this device (all devices if NULL) * * This function is mainly intended for use by layoutrecall. It attempts * to free the layout segment immediately, or else to mark it for return @@ -2663,7 +2718,8 @@ int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, struct list_head *tmp_list, const struct pnfs_layout_range *return_range, - u32 seq, bool cancel_io) + u32 seq, bool cancel_io, + const struct nfs4_deviceid *devid) { struct pnfs_layout_segment *lseg, *next; struct nfs_server *server = NFS_SERVER(lo->plh_inode); @@ -2690,7 +2746,7 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, remaining++; set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); if (cancel_io) - pnfs_lseg_cancel_io(server, lseg); + pnfs_lseg_cancel_io(server, lseg, devid); } if (remaining) { @@ -2708,7 +2764,8 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, static void pnfs_mark_layout_for_return(struct inode *inode, - const struct pnfs_layout_range *range) + const struct pnfs_layout_range *range, + const struct nfs4_deviceid *devid) { struct pnfs_layout_hdr *lo; bool return_now = false; @@ -2726,7 +2783,7 @@ pnfs_mark_layout_for_return(struct inode *inode, * for how it works. */ if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0, - true) != -EBUSY) { + true, devid) != -EBUSY) { const struct cred *cred; nfs4_stateid stateid; enum pnfs_iomode iomode; @@ -2743,7 +2800,8 @@ pnfs_mark_layout_for_return(struct inode *inode, } void pnfs_error_mark_layout_for_return(struct inode *inode, - struct pnfs_layout_segment *lseg) + struct pnfs_layout_segment *lseg, + const struct nfs4_deviceid *devid) { struct pnfs_layout_range range = { .iomode = lseg->pls_range.iomode, @@ -2751,7 +2809,7 @@ void pnfs_error_mark_layout_for_return(struct inode *inode, .length = NFS4_MAX_UINT64, }; - pnfs_mark_layout_for_return(inode, &range); + pnfs_mark_layout_for_return(inode, &range, devid); } EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); @@ -2841,7 +2899,7 @@ restart: pnfs_get_layout_hdr(lo); pnfs_set_plh_return_info(lo, range->iomode, 0); if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, - range, 0, true) != 0 || + range, 0, true, NULL) != 0 || !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { spin_unlock(&inode->i_lock); rcu_read_unlock(); @@ -2875,6 +2933,294 @@ pnfs_layout_return_unused_byclid(struct nfs_client *clp, &range); } +struct pnfs_reresolve_deviceid_args { + const struct pnfs_layoutdriver_type *ld; + const struct nfs4_deviceid *devid; + bool immediate; + struct list_head put_list; +}; + +static int pnfs_layout_reresolve_deviceid_byserver(struct nfs_server *server, + void *data) +{ + struct pnfs_reresolve_deviceid_args *args = data; + struct pnfs_layout_hdr *lo; + struct inode *inode; + + if (server->pnfs_curr_ld != args->ld) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + inode = lo->plh_inode; + if (!inode) + continue; + spin_lock(&inode->i_lock); + args->ld->reresolve_deviceid(lo, args->devid, args->immediate, + &args->put_list); + spin_unlock(&inode->i_lock); + } + rcu_read_unlock(); + return 0; +} + +/* + * Invoke @ld's reresolve_deviceid hook for @devid on every layout of @clp's + * servers, then drain the put_list once the locks are dropped. + */ +void +pnfs_layout_reresolve_deviceid_byclid(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid, + bool immediate) +{ + struct pnfs_reresolve_deviceid_args args = { + .ld = ld, + .devid = devid, + .immediate = immediate, + .put_list = LIST_HEAD_INIT(args.put_list), + }; + struct nfs4_deviceid_put *put, *tmp; + + if (!ld->reresolve_deviceid) + return; + + nfs_client_for_each_server(clp, + pnfs_layout_reresolve_deviceid_byserver, &args); + + list_for_each_entry_safe(put, tmp, &args.put_list, node) { + list_del(&put->node); + nfs4_put_deviceid_node(put->dev); + kfree(put); + } +} + +struct pnfs_deviceid_ref_args { + const struct pnfs_layoutdriver_type *ld; + const struct nfs4_deviceid *devid; + struct list_head *result; + bool found; +}; + +static int pnfs_layout_deviceid_referenced_byserver( + struct nfs_server *server, void *data) +{ + struct pnfs_deviceid_ref_args *args = data; + struct pnfs_layout_hdr *lo; + struct inode *inode; + + if (server->pnfs_curr_ld != args->ld) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + inode = lo->plh_inode; + if (!inode) + continue; + spin_lock(&inode->i_lock); + if (NFS_I(inode)->layout == lo && pnfs_layout_is_valid(lo) && + args->ld->layout_references_deviceid(lo, args->devid)) + args->found = true; + spin_unlock(&inode->i_lock); + if (args->found) + break; + } + rcu_read_unlock(); + return args->found; +} + +/* + * pnfs_layout_deviceid_referenced_byclid - does any live layout of + * @clp's servers using @ld still reference deviceid @devid? + */ +bool +pnfs_layout_deviceid_referenced_byclid(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid) +{ + struct pnfs_deviceid_ref_args args = { + .ld = ld, + .devid = devid, + }; + + if (!ld->layout_references_deviceid) + return false; + + nfs_client_for_each_server(clp, + pnfs_layout_deviceid_referenced_byserver, &args); + return args.found; +} + +static int pnfs_layout_collect_deviceid_refs_byserver( + struct nfs_server *server, void *data) +{ + struct pnfs_deviceid_ref_args *args = data; + struct nfs4_deviceid_ref *ref, *tmp; + struct pnfs_layout_hdr *lo; + struct inode *inode; + LIST_HEAD(putme); + bool matched; + int ret = 0; + + if (server->pnfs_curr_ld != args->ld) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + inode = lo->plh_inode; + if (!inode) + continue; + + spin_lock(&inode->i_lock); + matched = NFS_I(inode)->layout == lo && + pnfs_layout_is_valid(lo) && + args->ld->layout_references_deviceid(lo, args->devid); + if (!matched) { + spin_unlock(&inode->i_lock); + continue; + } + ref = kzalloc_obj(*ref, GFP_ATOMIC); + if (!ref) { + spin_unlock(&inode->i_lock); + ret = -ENOMEM; + break; + } + /* NFS_I()->layout == lo under i_lock means the refcount has + * not reached zero: pnfs_put_layout_hdr() decrements to zero + * and detaches in the same critical section. + */ + pnfs_get_layout_hdr(lo); + ref->lo = lo; + nfs4_stateid_copy(&ref->stateid, &lo->plh_stateid); + ref->cred = get_cred(lo->plh_lc_cred); + spin_unlock(&inode->i_lock); + + /* the pinned hdr does not hold the inode: grab it (and + * keep the superblock active) for use across RPCs + */ + ref->inode = nfs_igrab_and_active(inode); + if (!ref->inode) { + /* The layout may still name the deviceID, so report a + * partial list rather than silently shortening it. + * Defer the put: it can layoutreturn and sleep. + */ + list_add(&ref->node, &putme); + ret = -EAGAIN; + break; + } + list_add_tail(&ref->node, args->result); + } + rcu_read_unlock(); + + list_for_each_entry_safe(ref, tmp, &putme, node) { + list_del(&ref->node); + pnfs_put_layout_hdr(ref->lo); + put_cred(ref->cred); + kfree(ref); + } + return ret; +} + +/* + * Collect @clp's layouts referencing @devid onto @result as entries usable + * across sleeping RPCs; release with pnfs_layout_put_deviceid_refs(). + * A negative return means @result is only a partial set. + */ +int +pnfs_layout_collect_deviceid_refs(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid, + struct list_head *result) +{ + struct pnfs_deviceid_ref_args args = { + .ld = ld, + .devid = devid, + .result = result, + }; + + if (!ld->layout_references_deviceid) + return 0; + + return nfs_client_for_each_server(clp, + pnfs_layout_collect_deviceid_refs_byserver, &args); +} + +void +pnfs_layout_put_deviceid_refs(struct list_head *result) +{ + struct nfs4_deviceid_ref *ref, *tmp; + + list_for_each_entry_safe(ref, tmp, result, node) { + list_del(&ref->node); + put_cred(ref->cred); + pnfs_put_layout_hdr(ref->lo); + nfs_iput_and_deactive(ref->inode); + kfree(ref); + } +} + +/* + * Queue @id for the state manager's Section 18.40.4 recovery, + * dropping duplicates of an already-queued suspect. + */ +void pnfs_deviceid_delete_mark(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *id) +{ + struct nfs4_deviceid_delete *dd, *new; + + new = kzalloc_obj(*new, GFP_KERNEL); + if (!new) + return; /* lost notification; recovery waits for the next */ + new->ld = pnfs_find_layoutdriver(ld->id); + if (!new->ld) { + kfree(new); + return; + } + memcpy(&new->id, id, sizeof(new->id)); + + spin_lock(&clp->cl_lock); + list_for_each_entry(dd, &clp->cl_deviceid_deletes, list) { + if (dd->ld == new->ld && + !memcmp(&dd->id, &new->id, sizeof(dd->id))) { + spin_unlock(&clp->cl_lock); + pnfs_put_layoutdriver(new->ld); + kfree(new); + return; + } + } + list_add_tail(&new->list, &clp->cl_deviceid_deletes); + spin_unlock(&clp->cl_lock); + + set_bit(NFS4CLNT_DEVICEID_DELETE, &clp->cl_state); + nfs4_schedule_state_manager(clp); +} + +struct nfs4_deviceid_delete *pnfs_deviceid_delete_dequeue( + struct nfs_client *clp) +{ + struct nfs4_deviceid_delete *dd = NULL; + + spin_lock(&clp->cl_lock); + if (!list_empty(&clp->cl_deviceid_deletes)) { + dd = list_first_entry(&clp->cl_deviceid_deletes, + struct nfs4_deviceid_delete, list); + list_del(&dd->list); + } + spin_unlock(&clp->cl_lock); + return dd; +} + +void pnfs_deviceid_delete_queue_free(struct nfs_client *clp) +{ + struct nfs4_deviceid_delete *dd; + + while ((dd = pnfs_deviceid_delete_dequeue(clp)) != NULL) { + pnfs_put_layoutdriver(dd->ld); + kfree(dd); + } +} + /* Check if we have we have a valid layout but if there isn't an intersection * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. */ @@ -3103,6 +3449,7 @@ pnfs_do_write(struct nfs_pageio_descriptor *desc, static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) { + pnfs_put_ds_dev(hdr->ds_dev); pnfs_put_lseg(hdr->lseg); nfs_pgio_header_free(hdr); } @@ -3248,6 +3595,7 @@ pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) { + pnfs_put_ds_dev(hdr->ds_dev); pnfs_put_lseg(hdr->lseg); nfs_pgio_header_free(hdr); } diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 70d20f779678..2774d4adf4c3 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -57,7 +57,7 @@ struct nfs4_pnfs_ds_addr { }; struct nfs4_pnfs_ds { - struct list_head ds_node; /* nfs4_pnfs_dev_hlist dev_dslist */ + struct hlist_node ds_node; /* nfs_net nfs4_data_server_cache */ char *ds_remotestr; /* comma sep list of addrs */ struct list_head ds_addrs; const struct net *ds_net; @@ -171,6 +171,25 @@ struct pnfs_layoutdriver_type { struct nfs4_deviceid_node * (*alloc_deviceid_node) (struct nfs_server *server, struct pnfs_device *pdev, gfp_t gfp_flags); + /* + * Re-resolve @lo's references to the changed deviceid @id. Called + * under @lo's inode i_lock inside an RCU read-side critical section: + * must not sleep, allocations are GFP_ATOMIC. Rather than put the + * references it gives up (the final put can sleep), the hook + * allocates an nfs4_deviceid_put per reference and queues it on + * @put_list for the caller to put and free. On allocation failure + * it must leave the reference in place. + */ + void (*reresolve_deviceid)(struct pnfs_layout_hdr *lo, + const struct nfs4_deviceid *id, + bool immediate, + struct list_head *put_list); + /* + * Does @lo hold any reference to deviceid @id? Called under + * @lo's inode i_lock; must not sleep. + */ + bool (*layout_references_deviceid)(struct pnfs_layout_hdr *lo, + const struct nfs4_deviceid *id); int (*prepare_layoutreturn) (struct nfs4_layoutreturn_args *); @@ -178,7 +197,8 @@ struct pnfs_layoutdriver_type { int (*prepare_layoutcommit) (struct nfs4_layoutcommit_args *args); int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args); - void (*cancel_io)(struct pnfs_layout_segment *lseg); + void (*cancel_io)(struct pnfs_layout_segment *lseg, + const struct nfs4_deviceid *devid); }; struct pnfs_commit_ops { @@ -301,7 +321,8 @@ int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, struct list_head *tmp_list, const struct pnfs_layout_range *recall_range, - u32 seq, bool cancel_io); + u32 seq, bool cancel_io, + const struct nfs4_deviceid *devid); int pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, struct list_head *lseg_list); bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, @@ -351,9 +372,56 @@ int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *); int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *); struct nfs4_threshold *pnfs_mdsthreshold_alloc(void); void pnfs_error_mark_layout_for_return(struct inode *inode, - struct pnfs_layout_segment *lseg); + struct pnfs_layout_segment *lseg, + const struct nfs4_deviceid *devid); void pnfs_layout_return_unused_byclid(struct nfs_client *clp, enum pnfs_iomode iomode); +void pnfs_layout_reresolve_deviceid_byclid(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid, + bool immediate); +bool pnfs_layout_deviceid_referenced_byclid(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid); + +/* + * One live layout referencing a deviceID, collected for the + * CB_NOTIFY_DEVICEID DELETE recovery: the hdr is pinned, the inode + * igrab'd with its superblock active, and the layout stateid and + * cred snapshotted for TEST_STATEID. + */ +struct nfs4_deviceid_ref { + struct list_head node; + struct pnfs_layout_hdr *lo; + struct inode *inode; + nfs4_stateid stateid; + const struct cred *cred; +}; + +int pnfs_layout_collect_deviceid_refs(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *devid, + struct list_head *result); +void pnfs_layout_put_deviceid_refs(struct list_head *result); + +/* + * A CB_NOTIFY_DEVICEID DELETE naming a deviceID that live layouts + * still reference (RFC 8881 Section 18.40.4). Queued on + * nfs_client.cl_deviceid_deletes under cl_lock for the state manager + * to resolve; holds a layoutdriver reference. + */ +struct nfs4_deviceid_delete { + struct list_head list; + const struct pnfs_layoutdriver_type *ld; + struct nfs4_deviceid id; +}; + +void pnfs_deviceid_delete_mark(struct nfs_client *clp, + const struct pnfs_layoutdriver_type *ld, + const struct nfs4_deviceid *id); +struct nfs4_deviceid_delete *pnfs_deviceid_delete_dequeue( + struct nfs_client *clp); +void pnfs_deviceid_delete_queue_free(struct nfs_client *clp); int pnfs_layout_handle_reboot(struct nfs_client *clp); /* nfs4_deviceid_flags */ @@ -376,15 +444,32 @@ struct nfs4_deviceid_node { atomic_t ref; }; +/* One reference given up by reresolve_deviceid; nodes are shared, so a + * single pass can unpin the same node more than once. + */ +struct nfs4_deviceid_put { + struct list_head node; + struct nfs4_deviceid_node *dev; +}; + struct nfs4_deviceid_node * nfs4_find_get_deviceid(struct nfs_server *server, const struct nfs4_deviceid *id, const struct cred *cred, gfp_t gfp_mask); void nfs4_delete_deviceid(const struct pnfs_layoutdriver_type *, const struct nfs_client *, const struct nfs4_deviceid *); +void nfs4_deviceid_bump_change_epoch(struct nfs_client *clp); void nfs4_init_deviceid_node(struct nfs4_deviceid_node *, struct nfs_server *, const struct nfs4_deviceid *); bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *); void nfs4_mark_deviceid_available(struct nfs4_deviceid_node *node); + +/* Put the device node reference carried by an in-flight I/O, if any */ +static inline void pnfs_put_ds_dev(struct nfs4_deviceid_node *dev) +{ + if (dev) + nfs4_put_deviceid_node(dev); +} + void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node); bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node); void nfs4_deviceid_purge_client(const struct nfs_client *); @@ -422,11 +507,13 @@ struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, void nfs4_pnfs_v3_ds_connect_unload(void); int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, struct nfs4_deviceid_node *devid, unsigned int timeo, - unsigned int retrans, u32 version, u32 minor_version, + unsigned int retrans, unsigned int nconnect, + u32 version, u32 minor_version, bool tightly_coupled); struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags); +void nfs4_pnfs_ds_addr_list_free(struct list_head *dsaddrs); void pnfs_layout_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, struct nfs_commit_info *cinfo, @@ -690,10 +777,11 @@ pnfs_lseg_request_intersecting(struct pnfs_layout_segment *lseg, struct nfs_page } static inline void pnfs_lseg_cancel_io(struct nfs_server *server, - struct pnfs_layout_segment *lseg) + struct pnfs_layout_segment *lseg, + const struct nfs4_deviceid *devid) { if (server->pnfs_curr_ld->cancel_io) - server->pnfs_curr_ld->cancel_io(lseg); + server->pnfs_curr_ld->cancel_io(lseg, devid); } extern unsigned int layoutstats_timer; diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c index 274abdd6d5f3..e8143977965c 100644 --- a/fs/nfs/pnfs_dev.c +++ b/fs/nfs/pnfs_dev.c @@ -40,8 +40,11 @@ /* * Device ID RCU cache. A device ID is unique per server and layout type. + * + * 256 buckets keeps the chains short at the 1024-or-more devices a + * striping deployment expects, for 2KB of BSS on 64-bit. */ -#define NFS4_DEVICE_ID_HASH_BITS 5 +#define NFS4_DEVICE_ID_HASH_BITS 8 #define NFS4_DEVICE_ID_HASH_SIZE (1 << NFS4_DEVICE_ID_HASH_BITS) #define NFS4_DEVICE_ID_HASH_MASK (NFS4_DEVICE_ID_HASH_SIZE - 1) @@ -181,6 +184,21 @@ __nfs4_find_get_deviceid(struct nfs_server *server, return d; } +/* + * Bumped before the stale entry is unhashed, so an insert serialised + * after the unhash by nfs4_deviceid_lock observes the new epoch. + */ +void +nfs4_deviceid_bump_change_epoch(struct nfs_client *clp) +{ + atomic_inc(&clp->cl_deviceid_change_epoch); +} + +/* Discarding a raced reply is an optimisation, not a correctness + * requirement, and the epoch moves at the server's rate: bound it. + */ +#define NFS4_DEVICEID_FETCH_RETRIES 3 + struct nfs4_deviceid_node * nfs4_find_get_deviceid(struct nfs_server *server, const struct nfs4_deviceid *id, const struct cred *cred, @@ -188,11 +206,14 @@ nfs4_find_get_deviceid(struct nfs_server *server, { long hash = nfs4_deviceid_hash(id); struct nfs4_deviceid_node *d, *new; + int epoch, tries = 0; +retry: d = __nfs4_find_get_deviceid(server, id, hash); if (d) goto found; + epoch = atomic_read(&server->nfs_client->cl_deviceid_change_epoch); new = nfs4_get_device_info(server, id, cred, gfp_mask); if (!new) { trace_nfs4_find_deviceid(server, id, -ENOENT); @@ -200,6 +221,13 @@ nfs4_find_get_deviceid(struct nfs_server *server, } spin_lock(&nfs4_deviceid_lock); + if (atomic_read(&server->nfs_client->cl_deviceid_change_epoch) != epoch && + ++tries <= NFS4_DEVICEID_FETCH_RETRIES) { + /* a mapping changed while we fetched; ours may be stale */ + spin_unlock(&nfs4_deviceid_lock); + server->pnfs_curr_ld->free_deviceid_node(new); + goto retry; + } d = __nfs4_find_get_deviceid(server, id, hash); if (d) { spin_unlock(&nfs4_deviceid_lock); diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 93d63f75a355..7adb6f941cf2 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -15,6 +15,8 @@ #include "nfs4session.h" #include "internal.h" +#include +#include #include "pnfs.h" #include "netns.h" #include "nfs4trace.h" @@ -55,6 +57,7 @@ void pnfs_generic_commit_release(void *calldata) struct nfs_commit_data *data = calldata; data->completion_ops->completion(data); + pnfs_put_ds_dev(data->ds_dev); pnfs_put_lseg(data->lseg); nfs_put_client(data->ds_clp); nfs_commitdata_release(data); @@ -576,8 +579,8 @@ same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2) } /* - * Checks if 'dsaddrs1' contains a subset of 'dsaddrs2'. If it does, - * declare a match. + * Checks if 'dsaddrs1' and 'dsaddrs2' hold the same set of addresses. + * If they do, declare a match. */ static bool _same_data_server_addrs_locked(const struct list_head *dsaddrs1, @@ -587,6 +590,10 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, struct sockaddr *sa1, *sa2; bool match = false; + if (list_count_nodes((struct list_head *)dsaddrs1) != + list_count_nodes((struct list_head *)dsaddrs2)) + return false; + list_for_each_entry(da1, dsaddrs1, da_node) { sa1 = (struct sockaddr *)&da1->da_addr; match = false; @@ -602,16 +609,58 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, return match; } +/* Hash family, address bytes, and port - as same_sockaddr() */ +static u32 +nfs4_ds_addr_hash(const struct sockaddr *sa) +{ + u32 h = sa->sa_family; + + switch (sa->sa_family) { + case AF_INET: { + const struct sockaddr_in *a = (const struct sockaddr_in *)sa; + + h = jhash(&a->sin_addr.s_addr, sizeof(a->sin_addr.s_addr), h); + h = jhash(&a->sin_port, sizeof(a->sin_port), h); + break; + } + case AF_INET6: { + const struct sockaddr_in6 *a = (const struct sockaddr_in6 *)sa; + + h = jhash(&a->sin6_addr, sizeof(a->sin6_addr), h); + h = jhash(&a->sin6_port, sizeof(a->sin6_port), h); + break; + } + } + return h; +} + /* - * Lookup DS by addresses and NFS version. nfs4_ds_cache_lock is held + * Bucket index for a DS cache key. Per-address hashes combine by + * addition so the multipath list order cannot change the bucket, + * matching the order-independent set comparison above. + */ +static u32 +nfs4_ds_cache_hash(const struct list_head *dsaddrs, u32 version) +{ + const struct nfs4_pnfs_ds_addr *da; + u32 h = 0; + + list_for_each_entry(da, dsaddrs, da_node) + h += nfs4_ds_addr_hash((const struct sockaddr *)&da->da_addr); + return hash_32(jhash_1word(version, h), NFS4_DS_CACHE_HASH_BITS); +} + +/* + * Lookup DS by addresses and NFS version. nfs4_data_server_lock is held */ static struct nfs4_pnfs_ds * _data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs, u32 version) { struct nfs4_pnfs_ds *ds; + u32 bucket = nfs4_ds_cache_hash(dsaddrs, version); - list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node) + hlist_for_each_entry(ds, &nn->nfs4_data_server_cache[bucket], ds_node) if (ds->ds_version == version && _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) return ds; @@ -633,23 +682,28 @@ static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da) kfree(da); } -static void destroy_ds(struct nfs4_pnfs_ds *ds) +void nfs4_pnfs_ds_addr_list_free(struct list_head *dsaddrs) { struct nfs4_pnfs_ds_addr *da; + while (!list_empty(dsaddrs)) { + da = list_first_entry(dsaddrs, struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + nfs4_pnfs_ds_addr_free(da); + } +} +EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_addr_list_free); + +static void destroy_ds(struct nfs4_pnfs_ds *ds) +{ dprintk("--> %s\n", __func__); ifdebug(FACILITY) print_ds(ds); nfs_put_client(ds->ds_clp); - while (!list_empty(&ds->ds_addrs)) { - da = list_first_entry(&ds->ds_addrs, - struct nfs4_pnfs_ds_addr, - da_node); - list_del_init(&da->da_node); - nfs4_pnfs_ds_addr_free(da); - } + nfs4_pnfs_ds_addr_list_free(&ds->ds_addrs); kfree(ds->ds_remotestr); kfree(ds); @@ -660,7 +714,7 @@ void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds) struct nfs_net *nn = net_generic(ds->ds_net, nfs_net_id); if (refcount_dec_and_lock(&ds->ds_count, &nn->nfs4_data_server_lock)) { - list_del_init(&ds->ds_node); + hlist_del_init(&ds->ds_node); spin_unlock(&nn->nfs4_data_server_lock); destroy_ds(ds); } @@ -726,6 +780,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, { struct nfs_net *nn = net_generic(net, nfs_net_id); struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; + struct hlist_head *bucket; char *remotestr; if (list_empty(dsaddrs)) { @@ -739,6 +794,8 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, /* this is only used for debugging, so it's ok if its NULL */ remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); + /* @dsaddrs is empty after the splice below. */ + bucket = &nn->nfs4_data_server_cache[nfs4_ds_cache_hash(dsaddrs, version)]; spin_lock(&nn->nfs4_data_server_lock); tmp_ds = _data_server_lookup_locked(nn, dsaddrs, version); @@ -747,11 +804,11 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, list_splice_init(dsaddrs, &ds->ds_addrs); ds->ds_remotestr = remotestr; refcount_set(&ds->ds_count, 1); - INIT_LIST_HEAD(&ds->ds_node); + INIT_HLIST_NODE(&ds->ds_node); ds->ds_net = net; ds->ds_clp = NULL; ds->ds_version = version; - list_add(&ds->ds_node, &nn->nfs4_data_server_cache); + hlist_add_head(&ds->ds_node, bucket); dprintk("%s add new data server %s\n", __func__, ds->ds_remotestr); } else { @@ -787,7 +844,8 @@ static struct nfs_client *(*get_v3_ds_connect)( int ds_addrlen, int ds_proto, unsigned int ds_timeo, - unsigned int ds_retrans); + unsigned int ds_retrans, + unsigned int ds_nconnect); static bool load_v3_ds_connect(void) { @@ -810,7 +868,8 @@ void nfs4_pnfs_v3_ds_connect_unload(void) static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, unsigned int timeo, - unsigned int retrans) + unsigned int retrans, + unsigned int nconnect) { struct nfs_client *clp = ERR_PTR(-EIO); struct nfs_client *mds_clp = mds_srv->nfs_client; @@ -862,7 +921,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv, ds_proto = XPRT_TRANSPORT_TCP_TLS; clp = get_v3_ds_connect(mds_srv, &da->da_addr, da->da_addrlen, - ds_proto, timeo, retrans); + ds_proto, timeo, retrans, nconnect); if (IS_ERR(clp)) continue; clp->cl_rpcclient->cl_softerr = 0; @@ -885,6 +944,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, unsigned int timeo, unsigned int retrans, + unsigned int nconnect, u32 minor_version, bool tightly_coupled) { @@ -976,7 +1036,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, clp = nfs4_set_ds_client(mds_srv, &da->da_addr, da->da_addrlen, ds_proto, - timeo, retrans, minor_version, + timeo, retrans, nconnect, + minor_version, tightly_coupled); if (IS_ERR(clp)) continue; @@ -1011,7 +1072,8 @@ out: */ int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, struct nfs4_deviceid_node *devid, unsigned int timeo, - unsigned int retrans, u32 version, u32 minor_version, + unsigned int retrans, unsigned int nconnect, + u32 version, u32 minor_version, bool tightly_coupled) { int err; @@ -1031,11 +1093,13 @@ int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, switch (version) { case 3: - err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans); + err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans, + nconnect); break; case 4: err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo, retrans, - minor_version, tightly_coupled); + nconnect, minor_version, + tightly_coupled); break; default: dprintk("%s: unsupported DS version %d\n", __func__, version); diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c index 03c2c1f31be9..406d04b52a7d 100644 --- a/fs/nfs/proc.c +++ b/fs/nfs/proc.c @@ -323,9 +323,17 @@ nfs_proc_remove(struct inode *dir, struct dentry *dentry) .rpc_argp = &arg, }; int status; + struct inode *inode = d_inode(dentry); + unsigned long gencount; dprintk("NFS call remove %pd2\n",dentry); + if (inode) + gencount = READ_ONCE(NFS_I(inode)->attr_gencount); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + if (status == 0 && inode) + nfs_drop_nlink(inode, gencount); + nfs_mark_for_revalidate(dir); dprintk("NFS reply remove: %d\n", status); diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 2b70bd2b934b..e7497b029d6c 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -324,7 +324,7 @@ int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, aligned_len = min_t(unsigned int, ALIGN(len, rsize), fsize); - new = nfs_page_create_from_folio(ctx, folio, 0, aligned_len); + new = nfs_page_create_from_folio(ctx, folio, false, 0, aligned_len); if (IS_ERR(new)) { error = PTR_ERR(new); if (nfs_netfs_folio_unlock(folio)) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index b6967b528669..0b42d31f7a75 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1087,7 +1087,7 @@ static struct nfs_page *nfs_setup_write_request(struct nfs_open_context *ctx, req = nfs_try_to_update_request(folio, offset, bytes); if (req != NULL) goto out; - req = nfs_page_create_from_folio(ctx, folio, offset, bytes); + req = nfs_page_create_from_folio(ctx, folio, false, offset, bytes); if (IS_ERR(req)) goto out; nfs_inode_add_request(req); diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index 85aa03a7b020..1423a43dbd6e 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -24,6 +24,26 @@ static DEFINE_SPINLOCK(nfs_uuids_lock); * that is protected by nfs_uuids_lock. */ static LIST_HEAD(nfs_uuids); +/* Namespace registration order, protected by nfs_uuids_lock. */ +static u64 nfs_uuid_seq; + +/* The owner waits for temporary users before freeing its nfs_client. */ +static void nfs_uuid_get(nfs_uuid_t *nfs_uuid) +{ + refcount_inc(&nfs_uuid->refs); +} + +static void nfs_uuid_release(nfs_uuid_t *nfs_uuid) +{ + bool released; + + spin_lock(&nfs_uuid->lock); + released = refcount_dec_and_test(&nfs_uuid->refs); + wake_up_var_locked(&nfs_uuid->refs, &nfs_uuid->lock); + spin_unlock(&nfs_uuid->lock); + if (released) + complete(&nfs_uuid->released); +} /* * Lock ordering: @@ -43,6 +63,8 @@ void nfs_uuid_init(nfs_uuid_t *nfs_uuid) INIT_LIST_HEAD(&nfs_uuid->list); INIT_LIST_HEAD(&nfs_uuid->files); spin_lock_init(&nfs_uuid->lock); + refcount_set(&nfs_uuid->refs, 1); + init_completion(&nfs_uuid->released); nfs_uuid->nfs3_localio_probe_count = 0; } EXPORT_SYMBOL_GPL(nfs_uuid_init); @@ -50,7 +72,12 @@ EXPORT_SYMBOL_GPL(nfs_uuid_init); bool nfs_uuid_begin(nfs_uuid_t *nfs_uuid) { spin_lock(&nfs_uuid->lock); - if (rcu_access_pointer(nfs_uuid->net)) { + /* The caller retains the client while temporary UUID users drain. */ + wait_var_event_spinlock(&nfs_uuid->refs, + refcount_read(&nfs_uuid->refs) <= 1, + &nfs_uuid->lock); + if (rcu_access_pointer(nfs_uuid->net) || nfs_uuid->list_lock || + refcount_read(&nfs_uuid->refs) != 1) { /* This nfs_uuid is already in use */ spin_unlock(&nfs_uuid->lock); return false; @@ -77,7 +104,7 @@ void nfs_uuid_end(nfs_uuid_t *nfs_uuid) { if (!rcu_access_pointer(nfs_uuid->net)) { spin_lock(&nfs_uuid->lock); - if (!rcu_access_pointer(nfs_uuid->net)) { + if (!rcu_access_pointer(nfs_uuid->net) && !nfs_uuid->list_lock) { /* Not local, remove from nfs_uuids */ spin_lock(&nfs_uuids_lock); list_del_init(&nfs_uuid->list); @@ -113,28 +140,37 @@ void nfs_uuid_is_local(const uuid_t *uuid, struct list_head *list, spin_unlock(&nfs_uuids_lock); return; } + /* Drop the lookup lock before taking nfs_uuid->lock. */ + nfs_uuid_get(nfs_uuid); + spin_unlock(&nfs_uuids_lock); + + spin_lock(&nfs_uuid->lock); + spin_lock(&nfs_uuids_lock); + /* nfs_uuid_end() may have withdrawn the probe in the meantime. */ + if (nfs_uuid_lookup_locked(uuid) != nfs_uuid) + goto out_unlock; /* * We don't hold a ref on the net, but instead put * ourselves on @list (nn->local_clients) so the net * pointer can be invalidated. */ - spin_lock(list_lock); /* list_lock is nn->local_clients_lock */ - list_move(&nfs_uuid->list, list); - spin_unlock(list_lock); - - spin_unlock(&nfs_uuids_lock); - /* Once nfs_uuid is parented to @list, avoid global nfs_uuids_lock */ - spin_lock(&nfs_uuid->lock); - __module_get(mod); nfsd_mod = mod; + spin_lock(list_lock); /* list_lock is nn->local_clients_lock */ nfs_uuid->list_lock = list_lock; kref_get(&dom->ref); nfs_uuid->dom = dom; rcu_assign_pointer(nfs_uuid->net, net); + nfs_uuid->seq = ++nfs_uuid_seq; + list_move_tail(&nfs_uuid->list, list); + spin_unlock(list_lock); + +out_unlock: + spin_unlock(&nfs_uuids_lock); spin_unlock(&nfs_uuid->lock); + nfs_uuid_release(nfs_uuid); } EXPORT_SYMBOL_GPL(nfs_uuid_is_local); @@ -154,6 +190,10 @@ static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) struct nfs_file_localio *nfl; spin_lock(&nfs_uuid->lock); + /* A concurrent disabler can drop the lock while closing files. */ + wait_var_event_spinlock(&nfs_uuid->list_lock, + rcu_access_pointer(nfs_uuid->net) || + !nfs_uuid->list_lock, &nfs_uuid->lock); if (unlikely(!rcu_access_pointer(nfs_uuid->net))) { spin_unlock(&nfs_uuid->lock); return false; @@ -211,6 +251,7 @@ static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) } module_put(nfsd_mod); + wake_up_var_locked(&nfs_uuid->list_lock, &nfs_uuid->lock); spin_unlock(&nfs_uuid->lock); return true; @@ -223,22 +264,50 @@ void nfs_localio_disable_client(struct nfs_client *clp) } EXPORT_SYMBOL_GPL(nfs_localio_disable_client); +void nfs_localio_free_client(struct nfs_client *clp) +{ + nfs_uuid_t *nfs_uuid = &clp->cl_uuid; + + nfs_uuid_end(nfs_uuid); + nfs_localio_disable_client(clp); + /* Unlink before dropping the owner reference, then drain all users. */ + nfs_uuid_release(nfs_uuid); + wait_for_completion(&nfs_uuid->released); +} +EXPORT_SYMBOL_GPL(nfs_localio_free_client); + void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, spinlock_t *nn_local_clients_lock) { - LIST_HEAD(local_clients); - nfs_uuid_t *nfs_uuid, *tmp; + nfs_uuid_t *nfs_uuid; struct nfs_client *clp; + u64 last_seq; spin_lock(nn_local_clients_lock); - list_splice_init(nn_local_clients, &local_clients); - spin_unlock(nn_local_clients_lock); - list_for_each_entry_safe(nfs_uuid, tmp, &local_clients, list) { - if (WARN_ON(nfs_uuid->list_lock != nn_local_clients_lock)) + if (list_empty(nn_local_clients)) + goto out_unlock; + /* New registrations go to the tail and belong to a later sweep. */ + last_seq = list_last_entry(nn_local_clients, nfs_uuid_t, list)->seq; + for (;;) { + nfs_uuid = list_first_entry_or_null(nn_local_clients, + nfs_uuid_t, list); + /* Use a signed difference to allow the sequence to wrap. */ + if (!nfs_uuid || (s64)(nfs_uuid->seq - last_seq) > 0) break; + if (WARN_ON_ONCE(nfs_uuid->list_lock != nn_local_clients_lock)) + break; + /* Teardown unlinks under this lock before dropping its reference. */ + nfs_uuid_get(nfs_uuid); + spin_unlock(nn_local_clients_lock); + clp = container_of(nfs_uuid, struct nfs_client, cl_uuid); nfs_localio_disable_client(clp); + nfs_uuid_release(nfs_uuid); + cond_resched(); + spin_lock(nn_local_clients_lock); } +out_unlock: + spin_unlock(nn_local_clients_lock); } EXPORT_SYMBOL_GPL(nfs_localio_invalidate_clients); diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 34d294774f8c..416c6f39f31d 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -74,6 +74,8 @@ struct nfs_client { u64 cl_clientid; /* constant */ nfs4_verifier cl_confirm; /* Clientid verifier */ unsigned long cl_state; + /* bumped on each CB_NOTIFY_DEVICEID CHANGE for this client */ + atomic_t cl_deviceid_change_epoch; spinlock_t cl_lock; @@ -101,6 +103,8 @@ struct nfs_client { /* The flags used for obtaining the clientid during EXCHANGE_ID */ u32 cl_exchange_flags; struct nfs4_session *cl_session; /* shared session */ + /* CB_NOTIFY_DEVICEID DELETE suspects, protected by cl_lock */ + struct list_head cl_deviceid_deletes; bool cl_preserve_clid; struct nfs41_server_owner *cl_serverowner; struct nfs41_server_scope *cl_serverscope; @@ -248,6 +252,10 @@ struct nfs_server { that are supported on this filesystem */ struct pnfs_layoutdriver_type *pnfs_curr_ld; /* Active layout driver */ + unsigned int lg_reply_sz; /* Learned LAYOUTGET reply + buffer size, when the layout + driver's default has proved + too small */ struct rpc_wait_queue roc_rpcwaitq; /* the following fields are protected by nfs_client->cl_lock */ diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 4b9a35dbc062..c38e4b380be5 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -38,6 +38,7 @@ enum { PG_REMOVE, /* page group sync bit in write path */ PG_CONTENDED1, /* Is someone waiting for a lock? */ PG_CONTENDED2, /* Is someone waiting for a lock? */ + PG_PINNED, /* page is pinned by GUP */ }; struct nfs_inode; @@ -58,6 +59,7 @@ struct nfs_page { struct nfs_page *wb_this_page; /* list of reqs for this page */ struct nfs_page *wb_head; /* head pointer for req list */ unsigned short wb_nio; /* Number of I/O attempts */ + unsigned int wb_nr_pinned; /* Number of pinned pages */ }; struct nfs_pgio_mirror; @@ -125,15 +127,17 @@ struct nfs_pageio_descriptor { extern struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, struct page *page, + bool pinned, unsigned int pgbase, loff_t offset, unsigned int count); extern struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, struct folio *folio, + bool pinned, unsigned int offset, unsigned int count); -extern void nfs_release_request(struct nfs_page *); - +void nfs_release_request(struct nfs_page *req); +void nfs_release_request_list(struct list_head *head); extern void nfs_pageio_init(struct nfs_pageio_descriptor *desc, struct inode *inode, diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 7ed8fdb930d6..c0e29b4dfa62 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1693,6 +1693,7 @@ struct nfs_pgio_header { struct nfs_client *ds_clp; /* pNFS data server */ u32 ds_commit_idx; /* ds index if ds_clp is set */ u32 pgio_mirror_idx;/* mirror index in pgio layer */ + struct nfs4_deviceid_node *ds_dev; /* device node ref held across the I/O */ }; struct nfs_mds_commit_info { @@ -1731,6 +1732,7 @@ struct nfs_commit_data { struct nfs_open_context *context; struct pnfs_layout_segment *lseg; struct nfs_client *ds_clp; /* pNFS data server */ + struct nfs4_deviceid_node *ds_dev; /* device node ref held across the commit */ int ds_commit_index; loff_t lwb; const struct rpc_call_ops *mds_ops; diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 8ce4d978a636..914246e3f342 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -8,8 +8,10 @@ #if IS_ENABLED(CONFIG_NFS_LOCALIO) +#include #include #include +#include #include #include #include @@ -37,6 +39,11 @@ struct nfs_file_localio; typedef struct { uuid_t uuid; unsigned nfs3_localio_probe_count; + /* Owner reference plus temporary references held by nfs_common */ + refcount_t refs; + struct completion released; + /* Namespace registration order, protected by list_lock */ + u64 seq; /* this struct is over a cacheline, avoid bouncing */ spinlock_t ____cacheline_aligned lock; struct list_head list; @@ -55,6 +62,7 @@ void nfs_uuid_is_local(const uuid_t *, struct list_head *, spinlock_t *, void nfs_localio_enable_client(struct nfs_client *clp); void nfs_localio_disable_client(struct nfs_client *clp); +void nfs_localio_free_client(struct nfs_client *clp); void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, spinlock_t *nn_local_clients_lock); @@ -127,6 +135,10 @@ static inline void nfs_localio_disable_client(struct nfs_client *clp) { } +static inline void nfs_localio_free_client(struct nfs_client *clp) +{ +} + #endif /* CONFIG_NFS_LOCALIO */ #endif /* __LINUX_NFSLOCALIO_H */ diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index a82045804d34..005730623f3b 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -145,6 +145,7 @@ struct xprtsec_parms { enum xprtsec_policies policy; /* authentication material */ + key_serial_t keyring_serial; key_serial_t cert_serial; key_serial_t privkey_serial; }; diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 1285f04cdac1..c0d5042f3b66 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -984,12 +984,16 @@ void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt) * @rqst: controlling RPC request * @srcp: points to RPC message payload in receive buffer * @copy_len: remaining length of receive buffer content - * @pad: Write chunk pad bytes needed (zero for pure inline) + * @writelist: bytes conveyed by the Write chunk (zero for pure inline) * * The upper layer has set the maximum number of bytes it can * receive in each component of rq_rcv_buf. These values are set in * the head.iov_len, page_len, tail.iov_len, and buflen fields. * + * For READ replies with a Write chunk, the payload is already in + * the page list. An overestimated head length and buflen can be + * reduced without moving reply data. + * * Unlike the TCP equivalent (xdr_partial_copy_from_skb), in * many cases this function simply updates iov_base pointers in * rq_rcv_buf to point directly to the received reply data, to @@ -998,13 +1002,16 @@ void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt) * Returns the count of bytes which had to be memcopied. */ static unsigned long -rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) +rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, + u32 writelist) { unsigned long fixup_copy_count; int i, npages, curlen; char *destp; struct page **ppages; int page_base; + int pad = writelist & 3; + unsigned int delta; /* The head iovec is redirected to the RPC reply message * in the receive buffer, to avoid a memcopy. @@ -1016,8 +1023,20 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) * head.iov_len bytes are copied into the page list. */ curlen = rqst->rq_rcv_buf.head[0].iov_len; - if (curlen > copy_len) + if (curlen > copy_len) { + /* These READ replies have no inline fields after the payload. + * Trim the head without moving the directly-placed data. + */ + if (writelist) { + delta = curlen - copy_len; + + rqst->rq_rcv_buf.head[0].iov_len = copy_len; + rqst->rq_private_buf.head[0].iov_len = copy_len; + rqst->rq_rcv_buf.buflen -= delta; + rqst->rq_private_buf.buflen -= delta; + } curlen = copy_len; + } srcp += curlen; copy_len -= curlen; @@ -1243,9 +1262,19 @@ rpcrdma_decode_msg(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep, /* Build the RPC reply's Payload stream in rqst->rq_rcv_buf */ base = (char *)xdr_inline_decode(xdr, 0); - rpclen = xdr_stream_remaining(xdr); + rpclen = rep->rr_hdrbuf.len - xdr_stream_pos(xdr); + if (writelist) + rpclen &= ~3; + if (unlikely(!rpclen)) + return -EIO; + if (writelist && rpclen < rqst->rq_rcv_buf.head[0].iov_len) { + rqst->rq_rcv_buf.head[0].iov_len = rpclen; + rqst->rq_private_buf.head[0].iov_len = rpclen; + } + r_xprt->rx_stats.fixup_copy_count += - rpcrdma_inline_fixup(rqst, base, rpclen, writelist & 3); + rpcrdma_inline_fixup(rqst, base, rpclen, + xdr_pad_size(writelist)); r_xprt->rx_stats.total_rdma_reply += writelist; return rpclen + xdr_align_size(writelist); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index be335eed329d..54957295a631 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -114,9 +114,9 @@ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) rpcrdma_ep_put(ep); } -/* Ensure xprt_force_disconnect() is invoked exactly once when a - * connection is closed or lost. (The important thing is it needs - * to be invoked "at least" once). +/* Ensure xprt_force_disconnect() is invoked at most once for an + * unexpected endpoint failure. Local endpoint teardown closes this + * gate before it can generate CM events and flushed completions. */ void rpcrdma_force_disconnect(struct rpcrdma_ep *ep) { @@ -587,6 +587,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) if (!ep) return; + /* Prevent teardown-generated events from requesting another close. */ + atomic_xchg(&ep->re_force_disconnect, 1); + id = ep->re_id; rc = rdma_disconnect(id); trace_xprtrdma_disconnect(r_xprt, rc); diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index b97c70c12f62..027b37de694b 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -356,35 +356,6 @@ xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) return want; } -static int -xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, - unsigned int *msg_flags, struct cmsghdr *cmsg, int ret) -{ - u8 content_type = tls_get_record_type(sock->sk, cmsg); - u8 level, description; - - switch (content_type) { - case 0: - break; - case TLS_RECORD_TYPE_DATA: - /* TLS sets EOR at the end of each application data - * record, even though there might be more frames - * waiting to be decrypted. - */ - *msg_flags &= ~MSG_EOR; - break; - case TLS_RECORD_TYPE_ALERT: - tls_alert_recv(sock->sk, msg, &level, &description); - ret = (level == TLS_ALERT_LEVEL_FATAL) ? - -EACCES : -EAGAIN; - break; - default: - /* discard this record type */ - ret = -EAGAIN; - } - return ret; -} - static int xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags) { @@ -402,16 +373,42 @@ xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags) .msg_control = &u, .msg_controllen = sizeof(u), }; + u8 level, description; int ret; iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1, alert_kvec.iov_len); ret = sock_recvmsg(sock, &msg, flags); - if (ret > 0) { - if (tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT) - iov_iter_revert(&msg.msg_iter, ret); - ret = xs_sock_process_cmsg(sock, &msg, msg_flags, &u.cmsg, - -EAGAIN); + /* put_cmsg() shrinks msg_controllen, so a short one means + * kTLS filled in u.cmsg. + */ + if (ret >= 0 && msg.msg_controllen < sizeof(u)) { + if (tls_get_record_type(sock->sk, &u.cmsg) != + TLS_RECORD_TYPE_ALERT) + return -EAGAIN; + /* RFC 8446 Section 5.1: a record with an Alert type carries + * exactly one message, and an alert is two octets. + * tls_alert_recv() reads both without checking the length. + * alert_kvec caps the count at two, so a longer record + * fills it as well. kTLS sets MSG_EOR only once the + * record has been drained. + */ + if (ret != sizeof(alert) || !(msg.msg_flags & MSG_EOR)) + return -EACCES; + iov_iter_revert(&msg.msg_iter, ret); + tls_alert_recv(sock->sk, &msg, &level, &description); + /* RFC 8446 Section 6: every alert but a closure alert is + * an error alert, whatever the legacy AlertLevel octet + * says. + */ + switch (description) { + case TLS_ALERT_DESC_CLOSE_NOTIFY: + case TLS_ALERT_DESC_USER_CANCELED: + ret = -EAGAIN; + break; + default: + ret = -EACCES; + } } return ret; } @@ -425,6 +422,10 @@ xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) ret = sock_recvmsg(sock, msg, flags); /* Handle TLS inband control message lazily */ if (msg->msg_flags & MSG_CTRUNC) { + /* TLS sets EOR at the end of each application data + * record, even though there might be more frames + * waiting to be decrypted. + */ msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR); if (ret == 0 || ret == -EIO) ret = xs_sock_recv_cmsg(sock, &msg->msg_flags, flags); @@ -2636,6 +2637,7 @@ static int xs_tls_handshake_sync(struct rpc_xprt *lower_xprt, struct xprtsec_par goto out_put_xprt; break; case RPC_XPRTSEC_TLS_X509: + args.ta_keyring = xprtsec->keyring_serial; args.ta_my_cert = xprtsec->cert_serial; args.ta_my_privkey = xprtsec->privkey_serial; rc = tls_client_hello_x509(&args, GFP_KERNEL);