From edeebbfb6e7b3c28b61bda164702b1433701e3b7 Mon Sep 17 00:00:00 2001 From: Markuss Broks Date: Thu, 20 Aug 2026 22:12:05 +0300 Subject: [PATCH 01/69] iommu/exynos: detect SysMMUs without BLOCK mode Newer SysMMU instances may not implement BLOCK mode: writing CTRL_BLOCK does not stop translation, and MMU_STATUS never reports a blocked state. The hardware advertises this in CAPA1 bit 15, which the vendor driver reads as MMU_CAPA1_NO_BLOCK_MODE; the SysMMUs on Exynos8835 are such instances. Signed-off-by: Markuss Broks Acked-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 874d05f4b396..0319da9fd831 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -143,6 +143,7 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CTRL_VM_FAULT_MODE_STALL BIT(3) #define CAPA0_CAPA1_EXIST BIT(11) #define CAPA1_VCR_ENABLED BIT(14) +#define CAPA1_NO_BLOCK_MODE BIT(15) /* common registers */ #define REG_MMU_CTRL 0x000 @@ -306,6 +307,7 @@ struct sysmmu_drvdata { /* v7 fields */ bool has_vcr; /* virtual machine control register */ + bool no_block; /* BLOCK mode not implemented */ }; #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg) @@ -511,6 +513,7 @@ static void __sysmmu_get_vcr(struct sysmmu_drvdata *data) u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1); data->has_vcr = capa1 & CAPA1_VCR_ENABLED; + data->no_block = capa1 & CAPA1_NO_BLOCK_MODE; } static void __sysmmu_get_version(struct sysmmu_drvdata *data) From 0c11e843972ece0aa190869114d0e84680813b51 Mon Sep 17 00:00:00 2001 From: Markuss Broks Date: Thu, 20 Aug 2026 22:12:06 +0300 Subject: [PATCH 02/69] iommu/exynos: fix the enable sequence for no-block SysMMUs __sysmmu_enable() writes CTRL_BLOCK before programming CFG, the page table base and the VM registers. CTRL_BLOCK has the enable bit set, so on hardware without BLOCK mode this write immediately starts translation with whatever FLPT base the registers hold: reset values on the first enable, a stale page table on re-enable. A master that is already emitting traffic at that point gets its transactions translated through that garbage. Keep the MMU disabled while it is being programmed on such hardware; the final CTRL_ENABLE write then brings it up with a consistent configuration in one step, which is also the same as the vendor driver sequence. Signed-off-by: Markuss Broks Acked-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 0319da9fd831..a3a59d8a4cf1 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -655,7 +655,12 @@ static void __sysmmu_enable(struct sysmmu_drvdata *data) __sysmmu_enable_clocks(data); spin_lock_irqsave(&data->lock, flags); - writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + /* + * On no-block hardware CTRL_BLOCK acts as a plain enable; keep the + * MMU disabled until it is fully programmed. + */ + if (!data->no_block) + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); __sysmmu_init_config(data); __sysmmu_set_ptbase(data, data->pgtable); __sysmmu_enable_vid(data); From e29fd6f4f0fcd07556a71aa74dbe17d37a36b818 Mon Sep 17 00:00:00 2001 From: Markuss Broks Date: Thu, 20 Aug 2026 22:12:07 +0300 Subject: [PATCH 03/69] iommu/exynos: fix TLB invalidation for no-block SysMMUs sysmmu_block() polls MMU_STATUS for the blocked state, which no-block hardware never reports, so on such SysMMUs it always fails, and both sysmmu_tlb_invalidate_entry() and sysmmu_tlb_invalidate_flpdcache() skip the invalidation entirely when blocking fails. On no-block hardware the invalidation registers are written with the MMU running; the vendor driver never blocks around invalidation. Do the same and write the invalidation directly. In the FLPD cache path no-block implies a v7+ SysMMU, hence the unconditional flush-all. Signed-off-by: Markuss Broks Acked-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index a3a59d8a4cf1..29b27c7e400a 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -685,7 +685,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, spin_lock_irqsave(&data->lock, flags); if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { clk_enable(data->clk_master); - if (sysmmu_block(data)) { + /* + * No-block hardware accepts invalidation writes while the + * MMU is running; it is also v7+, so flush-all covers the + * FLPD cache. + */ + if (data->no_block) { + __sysmmu_tlb_invalidate(data); + } else if (sysmmu_block(data)) { if (data->version >= MAKE_MMU_VER(5, 0)) __sysmmu_tlb_invalidate(data); else @@ -721,7 +728,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64); - if (sysmmu_block(data)) { + if (data->no_block) { + __sysmmu_tlb_invalidate_entry(data, iova, num_inv); + } else if (sysmmu_block(data)) { __sysmmu_tlb_invalidate_entry(data, iova, num_inv); sysmmu_unblock(data); } From 25e8d16a9c9639c34e665d9b01027d77b5fd08f7 Mon Sep 17 00:00:00 2001 From: Markuss Broks Date: Thu, 20 Aug 2026 22:12:08 +0300 Subject: [PATCH 04/69] iommu/exynos: decode the v7 fault transaction info The v7+ fault registers carry a transaction-info word alongside the faulting address, but it is only used to derive the read/write direction and then thrown away. Keep it and print it: AxID identifies which port inside the master issued the faulting transaction, which is the only way to tell apart the several DMA engines a single block can contain, and the remaining bits of the raw word are implementation defined and worth having in a fault report. Define the known fields of the word instead of open-coding the masks; this also names the direction bit the driver was already testing as a bare BIT(20). Gate the print on the variant having a fault_info register, so older SysMMUs are unaffected. Signed-off-by: Markuss Broks Acked-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 29b27c7e400a..0bca3662fdc2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -8,6 +8,7 @@ #define DEBUG #endif +#include #include #include #include @@ -144,6 +145,9 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CAPA0_CAPA1_EXIST BIT(11) #define CAPA1_VCR_ENABLED BIT(14) #define CAPA1_NO_BLOCK_MODE BIT(15) +#define FAULT_INFO_AXID GENMASK(15, 0) +#define FAULT_INFO_AXLEN GENMASK(19, 16) +#define FAULT_INFO_WRITE BIT(20) /* common registers */ #define REG_MMU_CTRL 0x000 @@ -194,6 +198,7 @@ struct sysmmu_fault { sysmmu_iova_t addr; /* IOVA address that caused fault */ const char *name; /* human readable fault name */ unsigned int type; /* fault type for report_iommu_fault() */ + u32 info; /* raw transaction info (v7+ only) */ }; struct sysmmu_v1_fault_info { @@ -360,7 +365,8 @@ static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data, fault->addr = readl(SYSMMU_REG(data, fault_va)); fault->name = sysmmu_v7_fault_names[itype % 4]; - fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + fault->type = (info & FAULT_INFO_WRITE) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + fault->info = info; return 0; } @@ -558,6 +564,12 @@ static void show_fault_information(struct sysmmu_drvdata *data, dev_name(data->master), fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE", fault->name, fault->addr); + /* AxID identifies the issuing port inside the master */ + if (data->variant->fault_info) + dev_err(data->sysmmu, "transaction info %#010x: AxID %#lx, AxLEN %lu\n", + fault->info, + FIELD_GET(FAULT_INFO_AXID, fault->info), + FIELD_GET(FAULT_INFO_AXLEN, fault->info)); dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); ent = section_entry(phys_to_virt(data->pgtable), fault->addr); dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); From 3069e8f3e1e9bdf007316eadae4d1777065e7b78 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Fri, 21 Aug 2026 13:21:00 +0200 Subject: [PATCH 05/69] iommu/riscv: Report cache coherency capability VFIO and iommufd require IOMMU_CAP_CACHE_COHERENCY because they use IOMMU_CACHE mappings and have no userspace cache-maintenance path. RISC-V page tables preserve physical memory attributes for IOMMU_CACHE mappings. Svpbmt marks mappings without IOMMU_CACHE as non-cacheable, but preserving physical attributes cannot make a noncoherent device path coherent. Report the capability only for devices marked DMA coherent. This limits userspace assignment to paths that can honor cacheable mappings. Signed-off-by: Andrew Jones Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index fe8e6d0f8a23..fbdca69c436e 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1444,6 +1445,16 @@ static struct iommu_group *riscv_iommu_device_group(struct device *dev) return generic_device_group(dev); } +static bool riscv_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return dev_is_dma_coherent(dev); + default: + return false; + } +} + static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); @@ -1505,6 +1516,7 @@ static void riscv_iommu_release_device(struct device *dev) static const struct iommu_ops riscv_iommu_ops = { .of_xlate = riscv_iommu_of_xlate, + .capable = riscv_iommu_capable, .identity_domain = &riscv_iommu_identity_domain, .blocked_domain = &riscv_iommu_blocking_domain, .release_domain = &riscv_iommu_blocking_domain, From f5ec5d9476c43874ccab647f8b44ba783da37421 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Sun, 23 Aug 2026 06:20:15 +0800 Subject: [PATCH 06/69] iommu/amd: Do not request ACS when IOMMU is not going to be initialized The AMD IOMMU Initialization State Machine has the following state transition diagram (only the very first states are showed, and the `IOMMU_' prefix is omitted): START_STATE | v [0] detect_ivrs() --> NOT_FOUND | ok v IVRS_DETECTED | v [1] amd_iommu_disabled? (amd_iommu=off) --> IOMMU_CMDLINE_DISABLED | no v [2] early_amd_iommu_init() | +-- [3] amd_iommu_detected? (!iommu=off && ...) -+ | yes | +-- ... --> IOMMU_INIT_ERROR <-------------------+ | ok v IOMMU_ACPI_FINISHED | v ... [0] always calls pci_request_acs() as long as there's a valid IVRS table and no Stoney Ridge graphics. This is not optimal as ACS is not required in an [amd_]iommu=off boot. In a normal boot, ACS is requested due to amd_iommu_detect() requesting IVRS_DETECTED. pci_request_acs+0x9/0x18 iommu_go_to_state+0x106/0x1a20 amd_iommu_detect+0x1c/0x50 pci_iommu_alloc+0x26/0x40 mm_core_init+0xa/0x120 start_kernel+0x527/0x7a0 x86_64_start_reservations+0x24/0x30 x86_64_start_kernel+0xd1/0xe0 common_startup_64+0x13e/0x158 This is intended to ensure ACS is requested before the PCI core initialization, or else a !CONFIG_IRQ_REMAP, nointremap or intremap=off boot would be broken. However, in an amd_iommu=off boot, the state machine still requests ACS at the exact same time, as amd_iommu_detect() has nothing to do with amd_iommu_disabled. Even worse, in an iommu=off boot, though amd_iommu_detect() bails out early, ACS is still requested due to amd_iommu_prepare() requesting IOMMU_ACPI_FINISHED, which is called by irq_remapping_prepare() thanks to CONFIG_X86_LOCAL_APIC (always set on X86_64) and CONFIG_IRQ_REMAP (enabled by defconfig), unless nointremap or intremap=off is also passed to cmdline. pci_request_acs+0x9/0x18 iommu_go_to_state+0x106/0x1a20 amd_iommu_prepare+0x15/0x40 irq_remapping_prepare+0x43/0x60 enable_IR_x2apic+0x22/0x190 x86_64_probe_apic+0xa/0x50 apic_intr_mode_init+0x70/0xd0 x86_late_time_init+0x28/0x40 start_kernel+0x6f9/0x7a0 ... In both cases, [2] is still gated due to the [1] or [3] check, so that IOMMU can be disabled per cmdline. Technically, it makes no sense to detect IVRS at all in an [amd_]iommu=off boot or if IOMMU is not supported due to platform settings. This is probably why amd_iommu_detect() bails out before requesting IVRS_DETECTED. Apparently only bailing out there is not sufficient, and the bailing-out paths should really have been parts of the state machine. Clean up the initialization routines by moving the bailing-out paths and [1] to the right place in the state machine (i.e., before [0]), and always requesting IVRS_DETECTED in amd_iommu_detect() to initialize the state machine early and properly. Note that the condition (iommu_detected && !gart_iommu_aperture) is dropped as it's dead code since the relevant code being removed. It meant to detect what commit 6631ee9d0099 ("x86, AMD IOMMU: add dma_ops initialization function") had introduced, but commit d7f077697533 ("x86/amd-iommu: Fall back to GART if initialization fails") has removed it. Later, commit 78013eaadf69 ("x86: remove the IOMMU table infrastructure") ensured AMD IOMMU being detected before Intel's and led to iommu_detected == gart_iommu_aperture in any case. Signed-off-by: Rong Zhang Reviewed-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 8a410d4aa370..975d77b501f3 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3476,6 +3476,8 @@ static void amd_iommu_apply_erratum_snp(void) #endif } +static bool amd_iommu_sme_check(void); + /**************************************************************************** * * AMD IOMMU Initialization State Machine @@ -3488,7 +3490,13 @@ static int __init state_next(void) switch (init_state) { case IOMMU_START_STATE: - if (!detect_ivrs()) { + if (no_iommu || amd_iommu_disabled) { + init_state = IOMMU_CMDLINE_DISABLED; + ret = -EINVAL; + } else if (!amd_iommu_sme_check()) { + init_state = IOMMU_INIT_ERROR; + ret = -EINVAL; + } else if (!detect_ivrs()) { init_state = IOMMU_NOT_FOUND; ret = -ENODEV; } else { @@ -3496,13 +3504,8 @@ static int __init state_next(void) } break; case IOMMU_IVRS_DETECTED: - if (amd_iommu_disabled) { - init_state = IOMMU_CMDLINE_DISABLED; - ret = -EINVAL; - } else { - ret = early_amd_iommu_init(); - init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; - } + ret = early_amd_iommu_init(); + init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; break; case IOMMU_ACPI_FINISHED: early_enable_iommus(); @@ -3701,12 +3704,6 @@ void __init amd_iommu_detect(void) { int ret; - if (no_iommu || (iommu_detected && !gart_iommu_aperture)) - goto disable_snp; - - if (!amd_iommu_sme_check()) - goto disable_snp; - ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); if (ret) goto disable_snp; From f6e14328620efb98f3c988db4dddcaf8cfbb3a94 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Sun, 23 Aug 2026 06:20:16 +0800 Subject: [PATCH 07/69] iommu/amd: Disallow implicit START_STATE => IVRS_DETECTED transition Xen PV intentionally skips calling amd_iommu_detect() in order to gate IOMMU initialization, as the Xen hypervisor has taken over the hardware. However, due to amd_iommu_prepare() requesting ACPI_FINISHED while setting up APIC, the state machine is still progressed, only to be stopped midway due to the `!amd_iommu_detected' check in early_amd_iommu_init(). This is worthless and fragile. It unnecessarily calls detect_ivrs(), which leads to extra overhead, i.e., getting the IVRS table and accessing the PCI config space. Disallow implicit START_STATE => IVRS_DETECTED transition, so that amd_iommu_prepare() and other paths no longer progress the state machine accidentally. This should also help prevent potential bugs if more housekeeping work is added to amd_iommu_detect() in the future. Signed-off-by: Rong Zhang Reviewed-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 975d77b501f3..e6dee2c78a1a 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3577,6 +3577,19 @@ static int __init iommu_go_to_state(enum iommu_init_state state) { int ret = -EINVAL; + /* + * Some essential housekeeping work is done by amd_iommu_detect(). + * Skipping calling it implies that the platform (e.g., Xen hypervisor) + * has taken over the hardware. Progressing the state machine in this + * case is worthless and fragile. + * + * There are several paths requesting later states, so disallow implicit + * START_STATE => IVRS_DETECTED transition to prevent these paths from + * accidentally progressing the state machine. + */ + if (init_state == IOMMU_START_STATE && state != IOMMU_IVRS_DETECTED) + goto out; + while (init_state != state) { if (init_state == IOMMU_NOT_FOUND || init_state == IOMMU_INIT_ERROR || @@ -3585,6 +3598,7 @@ static int __init iommu_go_to_state(enum iommu_init_state state) ret = state_next(); } +out: /* * SNP platform initilazation requires IOMMUs to be fully configured. * If the SNP support on IOMMUs has NOT been checked, simply mark SNP From 567e74ce4dfde40a3b814de6af4adbfbe4ba1f38 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Sun, 23 Aug 2026 06:20:17 +0800 Subject: [PATCH 08/69] iommu/amd: Remove ad-hoc checks that are never true The state transitions of the AMD IOMMU Initialization State Machine imply that some ad-hoc checks will never be true. In detail, they are only reachable if the previous state transitions have succeeded, but the conditions are only true if the previous state transitions have failed. Therefore, remove the redundant ad-hoc checks. Signed-off-by: Rong Zhang Reviewed-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index e6dee2c78a1a..5b1a443f3800 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -163,7 +163,6 @@ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; -static bool amd_iommu_detected; static bool amd_iommu_disabled __initdata; static bool amd_iommu_force_enable __initdata; static bool amd_iommu_irtcachedis; @@ -3228,9 +3227,6 @@ static int __init early_amd_iommu_init(void) acpi_status status; u8 efr_hats, max_vasize; - if (!amd_iommu_detected) - return -ENODEV; - status = acpi_get_table("IVRS", 0, &ivrs_base); if (status == AE_NOT_FOUND) return -ENODEV; @@ -3309,7 +3305,7 @@ static int __init early_amd_iommu_init(void) } /* Disable any previously enabled IOMMUs */ - if (!is_kdump_kernel() || amd_iommu_disabled) + if (!is_kdump_kernel()) disable_iommus(); if (amd_iommu_irq_remap) @@ -3404,12 +3400,6 @@ static __init void iommu_snp_enable(void) if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - /* SNP support required IOMMU to be ON */ - if (no_iommu) { - pr_warn("SNP: IOMMU disabled, SNP cannot be supported.\n"); - goto disable_snp; - } - amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP); /* * If SNP page mode 0 is not enabled, then SNP support requires that IOMMU @@ -3722,7 +3712,6 @@ void __init amd_iommu_detect(void) if (ret) goto disable_snp; - amd_iommu_detected = true; iommu_detected = 1; x86_init.iommu.iommu_init = amd_iommu_init; return; From 183629813e2c57bc6cd75ebb5e0414df775f1c5b Mon Sep 17 00:00:00 2001 From: Zhaoyu Liu Date: Sun, 23 Aug 2026 17:24:36 +0800 Subject: [PATCH 09/69] iommu/sva: remove iommu_sva_present flag The global iommu_sva_present bool merely mirrors !list_empty(&iommu_sva_mms) under the same mutex. Remove it and rely on list_for_each_entry to handle the empty list case, which is a no-op. Signed-off-by: Zhaoyu Liu Reviewed-by: Lu Baolu Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sva.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 9742cb593577..9827cb804363 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -11,7 +11,6 @@ #include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); -static bool iommu_sva_present; static LIST_HEAD(iommu_sva_mms); static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm); @@ -137,11 +136,9 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm goto out_free_domain; domain->users = 1; - if (list_empty(&iommu_mm->sva_domains)) { - if (list_empty(&iommu_sva_mms)) - iommu_sva_present = true; + if (list_empty(&iommu_mm->sva_domains)) list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms); - } + list_add(&domain->next, &iommu_mm->sva_domains); out: refcount_set(&handle->users, 1); @@ -182,11 +179,8 @@ void iommu_sva_unbind_device(struct iommu_sva *handle) iommu_detach_device_pasid(domain, dev, iommu_mm->pasid); if (--domain->users == 0) { list_del(&domain->next); - if (list_empty(&iommu_mm->sva_domains)) { + if (list_empty(&iommu_mm->sva_domains)) list_del(&iommu_mm->mm_list_elm); - if (list_empty(&iommu_sva_mms)) - iommu_sva_present = false; - } iommu_domain_free(domain); } @@ -334,8 +328,6 @@ void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) struct iommu_mm_data *iommu_mm; guard(mutex)(&iommu_sva_lock); - if (!iommu_sva_present) - return; list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm) mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end); From bf5bdfd1f8d4ad5c6020468d09b68b09c2adc5c2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 24 Aug 2026 08:58:21 +0000 Subject: [PATCH 10/69] iommu/amd: Force identity mode for selected GPUs only Certain AMD GPUs must always operate in IOMMU identity mode. This was previously enforced using a PASID check, which happened to work because these specific GPUs are PASID-capable. However, this approach incorrectly applies identity mode enforcement to all PASID-capable devices, not just the GPUs that require it. This made sense in the past because the domain allocation API (iommu_ops->domain_alloc()) only received the domain type, so the driver had no way to inspect device capabilities and pick the most suitable page table format (v1 or v2). With the recent driver enhancement to use domain_alloc_paging_flags() for all paging domain allocations, the driver can now inspect the device and flags directly and choose the appropriate page table type per device. Update amd_iommu_def_domain_type() to force identity mapping only for the specific GPUs that require it, via a new quirks_force_identity_mapping(). With this change, a system booting in DMA translation mode will now select: * Guest (v2) page table for PASID-capable devices * Host (v1) page table for non-PASID-capable devices Also drop the amd_iommu_snp_en check, as SNP enforces paging domain, which doesn't work with the identity requirement of these GPUs. Link: https://lore.kernel.org/all/20200824105415.21000-1-joro@8bytes.org/ Link: https://lore.kernel.org/linux-iommu/20260723061548.10187-1-vasant.hegde@amd.com/ Cc: Alex Deucher Cc: Mario Limonciello Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 45 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 56262f6b1f70..24ad9c796b21 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3162,6 +3162,26 @@ static bool amd_iommu_is_attach_deferred(struct device *dev) return dev_data->defer_attach; } +static bool quirks_force_identity_mapping(struct pci_dev *pdev) +{ + int class = pdev->class >> 8; + + /* AMD GPU vendor ID */ + if (pdev->vendor != PCI_VENDOR_ID_ATI) + return false; + + /* GPU class */ + if (class != PCI_CLASS_DISPLAY_VGA && + class != PCI_CLASS_DISPLAY_OTHER) + return false; + + if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI) + return false; + + /* It is the GPU in an APU, force identity domain */ + return true; +} + static int amd_iommu_def_domain_type(struct device *dev) { struct iommu_dev_data *dev_data; @@ -3170,20 +3190,23 @@ static int amd_iommu_def_domain_type(struct device *dev) if (!dev_data) return 0; + if (!dev_is_pci(dev)) + return 0; + /* Always use DMA domain for untrusted device */ - if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) + if (to_pci_dev(dev)->untrusted) return IOMMU_DOMAIN_DMA; - /* - * Do not identity map IOMMUv2 capable devices when: - * - memory encryption is active, because some of those devices - * (AMD GPUs) don't have the encryption bit in their DMA-mask - * and require remapping. - * - SNP is enabled, because it prohibits DTE[Mode]=0. - */ - if (pdev_pasid_supported(dev_data) && - !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && - !amd_iommu_snp_en) { + /* Apply device specific quirks */ + if (quirks_force_identity_mapping(to_pci_dev(dev))) { + /* + * When memory encryption is active, some of these devices + * don't have the encryption bit in their DMA-mask and + * require remapping. + */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + return 0; + return IOMMU_DOMAIN_IDENTITY; } From 4d520541fe443a27eec2e78ea5a82dadb227f6fa Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 2 Sep 2026 21:18:54 +0200 Subject: [PATCH 11/69] dt-bindings: iommu: dart: Support specifying the DMA aperture Apple DARTs are often connected directly to devices that expect only a portion of their address space to be used for DMA (for example, because other ranges are mapped directly to something else). Follow the "#iommu-cells = <4>" example from iommu.txt and allow specifying the DMA aperture via additional cells. Since DART is used on 64-bit Apple silicon systems use two cells to specify the start and length of the aperture. A common aperture starts at 0x100_0000_0000 and spans a 36-bit address space. This corresponds to the vm-base and vm-size properties on the Apple device tree side of things. As not all devices require this and to keep compatibility with existing device trees allow 1 and 5 as values for #iommu-cells. Signed-off-by: Janne Grunau Acked-by: Conor Dooley Signed-off-by: Joerg Roedel --- .../devicetree/bindings/iommu/apple,dart.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 47ec7fa52c3a..6efbe732b4ce 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -44,10 +44,12 @@ properties: Optional since not all IOMMUs are attached to a clock gate. '#iommu-cells': - const: 1 + enum: [ 1, 5 ] description: - Has to be one. The single cell describes the stream id emitted by - a master to the IOMMU. + Has to be one or five. The first cell describes the stream id emitted by + a master to the IOMMU. The optional second and third cell describe the + aperture start of a master's DMA window and fourth and fifth cell describe + the size of DMA window. power-domains: maxItems: 1 @@ -66,11 +68,11 @@ examples: compatible = "apple,t8103-dart"; reg = <0x82f80000 0x4000>; interrupts = <1 781 4>; - #iommu-cells = <1>; + #iommu-cells = <5>; }; master1 { - iommus = <&dart1 0>; + iommus = <&dart1 0 0x100 0x0 0x10 0x0>; }; - |+ From 42eed9bb9ca43287e5ec506d3e39a5f0bfb98dff Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 2 Sep 2026 21:18:55 +0200 Subject: [PATCH 12/69] iommu: apple-dart: Support specifying the DMA aperture in the DT Apple DARTs are often connected directly to devices that expect only a portion of their address space to be used for DMA (for example, because other ranges are mapped directly to something else). Devices can specify the start and end of the aperture via "iommus" args if the DART device node specifies "#iommu-cells = <5>. Aperture start and length are 64-bit values as apertures above and larger than 4GB are used. Limit compile testing to 64-bit architectures to avoid warnings in the calculation of 64-bit dma_addr_t values. This range *can* be outside of the DART's IAS. In that case, it is assumed that the hardware truncates addresses and the page tables will only map the lower bits of the address. However, the specified range cannot straddle an IAS boundary (you cannot cover more than IAS worth of address space nor wrap). This corresponds to the vm-base and vm-size properties on the Apple device tree side of things. Co-developed-by: Hector Martin Signed-off-by: Hector Martin Reviewed-by: Sven Peter Signed-off-by: Janne Grunau Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 2 +- drivers/iommu/apple-dart.c | 74 +++++++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd69467a..27954f462582 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -303,7 +303,7 @@ config SPAPR_TCE_IOMMU config APPLE_DART tristate "Apple DART IOMMU Support" - depends on ARCH_APPLE || COMPILE_TEST + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART select IOMMU_API select IOMMU_IO_PGTABLE_DART diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b160fb464c5f..ebf4547d32b4 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -267,6 +268,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -285,6 +287,13 @@ struct apple_dart_master_cfg { /* Intersection of DART capabilitles */ u32 supports_bypass : 1; + /* + * DMA aperture start and end to be overridden by "iommus"' phandle + * args. By default determined by DART's ias but may be outside of it. + */ + dma_addr_t dma_min; + dma_addr_t dma_max; + struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -537,7 +546,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -551,8 +560,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -563,7 +572,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -590,6 +600,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + u32 ias = min_t(u32, dart->ias, fls64(cfg->dma_max)); int ret = 0; int i, j; @@ -610,7 +621,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, @@ -623,10 +634,10 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } + dart_domain->mask = DMA_BIT_MASK(pgtbl_cfg.ias); dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = 0; - dart_domain->domain.geometry.aperture_end = - (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + dart_domain->domain.geometry.aperture_start = cfg->dma_min; + dart_domain->domain.geometry.aperture_end = cfg->dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->finalized = true; @@ -803,20 +814,65 @@ static int apple_dart_of_xlate(struct device *dev, struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct apple_dart *dart = platform_get_drvdata(iommu_pdev); struct apple_dart *cfg_dart; + dma_addr_t dma_max = DMA_BIT_MASK(dart->ias); + dma_addr_t dma_min = 0; int i, sid; put_device(&iommu_pdev->dev); - if (args->args_count != 1) + if (args->args_count != 1 && args->args_count != 5) return -EINVAL; + sid = args->args[0]; + if (args->args_count == 5) { + dma_addr_t length = ((dma_addr_t)args->args[3] << 32) | args->args[4]; + + if (!length) + return -EINVAL; + + dma_min = ((dma_addr_t)args->args[1] << 32) | args->args[2]; + + if (!IS_ALIGNED(dma_min, dart->pgsize) || + !IS_ALIGNED(length, dart->pgsize)) { + dev_err(dev, "Unaligned DMA window %pad, %pad (0x%x)\n", + &dma_min, &length, dart->pgsize); + return -EINVAL; + } + if (check_add_overflow(dma_min, length - 1, &dma_max)) { + dev_err(dev, "DMA window length (%pad) overflows range for start %pad\n", + &length, &dma_min); + return -EINVAL; + } + + /* + * Ensure that the DMA window does not exceed the DART's ias. + */ + if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(dev, "Invalid DMA window for ias=%d\n", + dart->ias); + return -EINVAL; + } + } + if (!cfg) { cfg = kzalloc_obj(*cfg); if (!cfg) return -ENOMEM; /* Will be ANDed with DART capabilities */ cfg->supports_bypass = true; + /* Will be merged with other DARTs to the common range. */ + cfg->dma_min = dma_min; + cfg->dma_max = dma_max; + } else { + if (dma_min >= cfg->dma_max || cfg->dma_min >= dma_max) { + dev_err(dev, "non-overlapping DMA windows: %pad..%pad, %pad..%pad\n", + &dma_min, &dma_max, + &cfg->dma_min, &cfg->dma_max); + return -EINVAL; + } + cfg->dma_min = max(dma_min, cfg->dma_min); + cfg->dma_max = min(dma_max, cfg->dma_max); } dev_iommu_priv_set(dev, cfg); From 15d5b3027d9f823754911b0f9c3fd05f598fbc62 Mon Sep 17 00:00:00 2001 From: Pranjal Shrivastava Date: Thu, 10 Sep 2026 14:26:52 +0000 Subject: [PATCH 13/69] iommu/amd: Refactor device probe and capability initialization Restructure the device probe path to improve readability and prepare for cleaner error handling. Refactor check_device() into lookup_device to explicitly validate and return the amd_iommu ptr & devid. Refactor iommu_init_device() to return the allocated dev_data. Consolidate all PCI cap inits (MSI domains, PASID, ATS) into a new helper: iommu_init_device_caps(). Suggested-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Reviewed-by: Vasant Hegde Signed-off-by: Pranjal Shrivastava Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 131 +++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 65 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 24ad9c796b21..fb98f909cdbf 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -675,7 +675,8 @@ static void pdev_disable_caps(struct pci_dev *pdev) * This function checks if the driver got a valid device from the caller to * avoid dereferencing invalid pointers. */ -static bool check_device(struct device *dev) +static bool lookup_device(struct device *dev, + struct amd_iommu **iommu_out, u16 *devid_out) { struct amd_iommu_pci_seg *pci_seg; struct amd_iommu *iommu; @@ -690,7 +691,7 @@ static bool check_device(struct device *dev) devid = PCI_SBDF_TO_DEVID(sbdf); iommu = rlookup_amd_iommu(dev); - if (!iommu) + if (!iommu || !iommu->iommu.ops) return false; /* Out of our scope? */ @@ -698,47 +699,33 @@ static bool check_device(struct device *dev) if (devid > pci_seg->last_bdf) return false; + *iommu_out = iommu; + *devid_out = devid; return true; } -static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) +static struct iommu_dev_data *iommu_init_device(struct amd_iommu *iommu, + struct device *dev, u16 devid) { struct iommu_dev_data *dev_data; - int devid, sbdf; - if (dev_iommu_priv_get(dev)) - return 0; - - sbdf = get_device_sbdf_id(dev); - if (sbdf < 0) - return sbdf; - - devid = PCI_SBDF_TO_DEVID(sbdf); dev_data = find_dev_data(iommu, devid); if (!dev_data) - return -ENOMEM; + return ERR_PTR(-ENOMEM); dev_data->dev = dev; /* - * The dev_iommu_priv_set() needes to be called before setup_aliases. + * The dev_iommu_priv_set() needs to be called before setup_aliases. * Otherwise, subsequent call to dev_iommu_priv_get() will fail. */ dev_iommu_priv_set(dev, dev_data); setup_aliases(iommu, dev); - /* - * By default we use passthrough mode for IOMMUv2 capable device. - * But if amd_iommu=force_isolation is set (e.g. to debug DMA to - * invalid address), we ignore the capability for the device so - * it'll be forced to go into translation mode. - */ - if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && - dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) { - dev_data->flags = pdev_get_caps(to_pci_dev(dev)); - } + /* Wait for DTE updates to go through */ + iommu_completion_wait(iommu); - return 0; + return dev_data; } static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) @@ -2484,49 +2471,73 @@ out: mutex_unlock(&dev_data->mutex); } -static struct iommu_device *amd_iommu_probe_device(struct device *dev) +static void iommu_init_device_caps(struct iommu_dev_data *dev_data, + struct device *dev, + struct amd_iommu *iommu) { - struct iommu_device *iommu_dev; - struct amd_iommu *iommu; - struct iommu_dev_data *dev_data; - int ret; - - if (!check_device(dev)) - return ERR_PTR(-ENODEV); - - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return ERR_PTR(-ENODEV); - - /* Not registered yet? */ - if (!iommu->iommu.ops) - return ERR_PTR(-ENODEV); - - if (dev_iommu_priv_get(dev)) - return &iommu->iommu; - - ret = iommu_init_device(iommu, dev); - if (ret) { - dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); - iommu_dev = ERR_PTR(ret); - iommu_ignore_device(iommu, dev); - goto out_err; - } + if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) + dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; + else + dev_data->max_irqs = MAX_IRQS_PER_TABLE_512; amd_iommu_set_pci_msi_domain(dev, iommu); - iommu_dev = &iommu->iommu; + + if (!dev_is_pci(dev)) + return; + + /* + * By default we use passthrough mode for IOMMUv2 capable device. + * But if amd_iommu=force_isolation is set (e.g. to debug DMA to + * invalid address), we ignore the capability for the device so + * it'll be forced to go into translation mode. + */ + if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && + amd_iommu_gt_ppr_supported()) { + dev_data->flags = pdev_get_caps(to_pci_dev(dev)); + } /* * If IOMMU and device supports PASID then it will contain max * supported PASIDs, else it will be zero. */ - dev_data = dev_iommu_priv_get(dev); - if (amd_iommu_pasid_supported() && dev_is_pci(dev) && + if (amd_iommu_pasid_supported() && pdev_pasid_supported(dev_data)) { dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, pci_max_pasids(to_pci_dev(dev))); } + pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); +} + +static struct iommu_device *amd_iommu_probe_device(struct device *dev) +{ + struct iommu_device *iommu_dev; + struct amd_iommu *iommu; + struct iommu_dev_data *dev_data; + u16 devid; + + if (!lookup_device(dev, &iommu, &devid)) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) + return &iommu->iommu; + + dev_data = iommu_init_device(iommu, dev, devid); + if (IS_ERR(dev_data)) { + dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); + iommu_dev = ERR_CAST(dev_data); + iommu_ignore_device(iommu, dev); + goto out_err; + } + + iommu_init_device_caps(dev_data, dev, iommu); + iommu_dev = &iommu->iommu; + + /* + * When DMA translation is unavailable return error so the iommu core + * won't attempt domain attach for this device, while preserving its + * rlookup entry for interrupt remapping. + */ if (amd_iommu_pgtable == PD_MODE_NONE) { pr_warn_once("%s: DMA translation not supported by iommu.\n", __func__); @@ -2534,16 +2545,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) goto out_err; } - iommu_completion_wait(iommu); - - if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) - dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; - else - dev_data->max_irqs = MAX_IRQS_PER_TABLE_512; - - if (dev_is_pci(dev)) - pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); - out_err: return iommu_dev; } From 5b33c19a31c0839f788c03f777703868faacc77c Mon Sep 17 00:00:00 2001 From: Pranjal Shrivastava Date: Thu, 10 Sep 2026 14:26:53 +0000 Subject: [PATCH 14/69] iommu/amd: Remove iommu_ignore_device() The iommu_ignore_device() helper was historically called on device initialization failure to clear the primary Device Table Entry (DTE) via memset() and nullify the rlookup_table entry. However, clearing the DTE on probe failure is problematic: 1. During normal boot, DTEs start out unconfigured (blocking DMA), making clearing redundant. 2. During kdump boot, pre-existing translations should be kept running until deferred attach rather than abruptly clearing them, which risks breaking in-flight transfers. 3. Writing to the DTE table via memset without flushing the hardware DTE cache risks aliasing & torn writes. 4. Clearing the rlookup_table entry breaks interrupt remapping for devices that fail probe or operate in translation-less modes. Remove iommu_ignore_device() entirely and simplify the error return paths in amd_iommu_probe_device(). Suggested-by: Jason Gunthorpe Suggested-by: Vasant Hegde Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260529153216.2AD1E1F00899@smtp.kernel.org/ Signed-off-by: Pranjal Shrivastava Reviewed-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index fb98f909cdbf..3a3cac6f6cdb 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -728,22 +728,6 @@ static struct iommu_dev_data *iommu_init_device(struct amd_iommu *iommu, return dev_data; } -static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) -{ - struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; - struct dev_table_entry *dev_table = get_dev_table(iommu); - int devid, sbdf; - - sbdf = get_device_sbdf_id(dev); - if (sbdf < 0) - return; - - devid = PCI_SBDF_TO_DEVID(sbdf); - pci_seg->rlookup_table[devid] = NULL; - memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); - - setup_aliases(iommu, dev); -} /**************************************************************************** @@ -2525,9 +2509,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) dev_data = iommu_init_device(iommu, dev, devid); if (IS_ERR(dev_data)) { dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); - iommu_dev = ERR_CAST(dev_data); - iommu_ignore_device(iommu, dev); - goto out_err; + return ERR_CAST(dev_data); } iommu_init_device_caps(dev_data, dev, iommu); @@ -2541,11 +2523,9 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) if (amd_iommu_pgtable == PD_MODE_NONE) { pr_warn_once("%s: DMA translation not supported by iommu.\n", __func__); - iommu_dev = ERR_PTR(-ENODEV); - goto out_err; + return ERR_PTR(-ENODEV); } -out_err: return iommu_dev; } From 622384f54a8415f28bf7cf11cda95ab125488ca1 Mon Sep 17 00:00:00 2001 From: Pranjal Shrivastava Date: Thu, 10 Sep 2026 14:26:54 +0000 Subject: [PATCH 15/69] iommu/amd: Fail probe on ATS configuration failure Update the driver to call pci_prepare_ats() after checking if pci_ats_supported() and fail the probe_device if pci_prepare_ats() returns an error. Additionally, update pdev_enable_cap_ats() to WARN_ON() a failure in pci_enable_ats(). Reviewed-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Reviewed-by: Samiullah Khawaja Signed-off-by: Pranjal Shrivastava Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 3a3cac6f6cdb..b6838fe6ebc1 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -573,10 +573,17 @@ static inline int pdev_enable_cap_ats(struct pci_dev *pdev) if (amd_iommu_iotlb_sup && (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { ret = pci_enable_ats(pdev, PAGE_SHIFT); - if (!ret) { - dev_data->ats_enabled = 1; - dev_data->ats_qdep = pci_ats_queue_depth(pdev); - } + + /* + * pci_enable_ats() should not fail here because earlier + * checks have already verified support & config. + */ + if (WARN_ON(ret)) + return ret; + + dev_data->ats_enabled = 1; + dev_data->ats_qdep = pci_ats_queue_depth(pdev); + ret = 0; } return ret; @@ -2455,10 +2462,12 @@ out: mutex_unlock(&dev_data->mutex); } -static void iommu_init_device_caps(struct iommu_dev_data *dev_data, - struct device *dev, - struct amd_iommu *iommu) +static int iommu_init_device_caps(struct iommu_dev_data *dev_data, + struct device *dev, + struct amd_iommu *iommu) { + int ret; + if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; else @@ -2467,7 +2476,7 @@ static void iommu_init_device_caps(struct iommu_dev_data *dev_data, amd_iommu_set_pci_msi_domain(dev, iommu); if (!dev_is_pci(dev)) - return; + return 0; /* * By default we use passthrough mode for IOMMUv2 capable device. @@ -2490,7 +2499,13 @@ static void iommu_init_device_caps(struct iommu_dev_data *dev_data, pci_max_pasids(to_pci_dev(dev))); } - pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); + if (pci_ats_supported(to_pci_dev(dev))) { + ret = pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); + if (ret) + return ret; + } + + return 0; } static struct iommu_device *amd_iommu_probe_device(struct device *dev) @@ -2499,6 +2514,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) struct amd_iommu *iommu; struct iommu_dev_data *dev_data; u16 devid; + int ret; if (!lookup_device(dev, &iommu, &devid)) return ERR_PTR(-ENODEV); @@ -2512,7 +2528,10 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) return ERR_CAST(dev_data); } - iommu_init_device_caps(dev_data, dev, iommu); + ret = iommu_init_device_caps(dev_data, dev, iommu); + if (ret) + return ERR_PTR(ret); + iommu_dev = &iommu->iommu; /* From aa416fa911ad67090291e39432560091f6ae9711 Mon Sep 17 00:00:00 2001 From: Federico Parola Date: Thu, 10 Sep 2026 15:04:47 +0000 Subject: [PATCH 16/69] iommu: Finalize deferred attachments when mapping MSI pages When a device's attachment to its default domain is deferred, the IOMMU is still translating the device with the tables inherited from the previous kernel. Any mapping installed in that domain does not take effect until the deferred attachment is finalized. iommu_deferred_attach() covers the DMA mapping paths, but nothing covers iommu_dma_prepare_msi(): programming an MSI installs the MSI page in group->domain and hands the resulting IOVA to the irqchip, so an MSI programmed before the first DMA map is written using an IOVA that the hardware does not yet translate. This issue is currently not triggerable as the two IOMMU implementations supporting deferred attachments, Intel and AMD, do not rely on DMA translations for MSI transactions. However, the Arm implementation being introduced in [1] will be subject to it. Finalize the deferred attachment from iommu_dma_prepare_msi() too, before the MSI page is mapped. Do it outside group->mutex, as iommu_deferred_attach() takes the mutex itself and the two operations do not need to be atomic with respect to each other: if an attach replaces group->domain in between, the MSI page is mapped into whatever group->domain holds once the mutex is taken. iommu_get_domain_for_dev() cannot return NULL here: MSI setup only runs for a device whose driver is bound, and of_iommu_configure() and acpi_iommu_configure_id() hold the bind off until the IOMMU is registered, hence until iommu_device_register() has set up the group's default domain. group->domain then stays non-NULL until iommu_deinit_device() clears it once the group is empty. dma_iova_try_alloc() already relies on the same guarantee. The static key gating the iommu_deferred_attach() calls in dma-iommu.c is private to that file, and replicating it here is not worthwhile since MSI page mapping is not a fast path. The behaviour of a rejected attach changes slightly: an attach refused because the device is being reset (-EBUSY) now fails MSI setup rather than only the first DMA map. Link: https://lore.kernel.org/linux-iommu/cover.1788130528.git.nicolinc@nvidia.com/ [1] Assisted-by: Claude:claude-opus-5 Signed-off-by: Federico Parola Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index cd1bca7ede9a..b486b8bbd1fc 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -4243,6 +4243,10 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) if (!group) return 0; + ret = iommu_deferred_attach(dev, iommu_get_domain_for_dev(dev)); + if (ret) + return ret; + mutex_lock(&group->mutex); /* An IDENTITY domain must pass through */ if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) { From f4eab1f88f792a5fb5965613551cd08071f7ceb8 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Fri, 11 Sep 2026 10:43:17 +0000 Subject: [PATCH 17/69] iommu/amd: Move GA log allocation to init Currently GA log buffer allocation is done at the time of enabling ga log. This mixes capability checking, resource allocation, and hardware enablement into a single late stage code path. Move capability checking and buffer allocation to init path: - check_vapic_support(): Checks AVIC support - alloc_ga_log(): Rename iommu_init_ga_log() as alloc_ga_log(). It will allocate the GA log and GA log tail buffers for a given IOMMU. Remove redundant checks inside alloc_ga_log(). - enable_iommus_vapic() is simplified - it handles enabling GA log and GAM/SNPAVIC support for IOMMUs. Finally update iommu_ga_log_enable() return value from int to bool as return value is used in boolean context only. Signed-off-by: Vasant Hegde Reviewed-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 74 +++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 5b1a443f3800..3c02ebd4ed22 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -915,13 +915,13 @@ static void free_ga_log(struct amd_iommu *iommu) } #ifdef CONFIG_IRQ_REMAP -static int iommu_ga_log_enable(struct amd_iommu *iommu) +static bool iommu_ga_log_enable(struct amd_iommu *iommu) { u32 status, i; u64 entry; if (!iommu->ga_log) - return -EINVAL; + return false; entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, @@ -945,35 +945,31 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) } if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) - return -EINVAL; + return false; - return 0; + return true; } +#endif /* CONFIG_IRQ_REMAP */ -static int iommu_init_ga_log(struct amd_iommu *iommu) +static int alloc_ga_log(struct amd_iommu *iommu) { +#ifdef CONFIG_IRQ_REMAP int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; - if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) - return -EINVAL; - - if (iommu->ga_log && iommu->ga_log_tail) - return 0; - iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) - goto err_out; + return -ENOMEM; iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8); - if (!iommu->ga_log_tail) - goto err_out; + if (!iommu->ga_log_tail) { + iommu_free_pages(iommu->ga_log); + iommu->ga_log = NULL; + return -ENOMEM; + } +#endif return 0; -err_out: - free_ga_log(iommu); - return -EINVAL; } -#endif /* CONFIG_IRQ_REMAP */ static int __init alloc_cwwb_sem(struct amd_iommu *iommu) { @@ -2151,6 +2147,27 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu) } } +/* Must be called after SNP support check is complete (iommu_snp_enable()) */ +static bool check_vapic_support(void) +{ + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + return false; + + if (!check_feature(FEATURE_GAM_VAPIC)) { + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; + return false; + } + + if (amd_iommu_snp_en && + !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { + pr_warn("Force to disable Virtual APIC due to SNP\n"); + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; + return false; + } + + return true; +} + static int __init iommu_init_pci(struct amd_iommu *iommu) { int cap_ptr = iommu->cap_ptr; @@ -2195,6 +2212,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) return -ENOMEM; + if (check_vapic_support()) { + ret = alloc_ga_log(iommu); + if (ret) + return ret; + } + if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { pr_info("Using strict mode due to virtualization\n"); iommu_set_dma_strict(); @@ -3009,22 +3032,9 @@ static void enable_iommus_vapic(void) if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return; - if (!check_feature(FEATURE_GAM_VAPIC)) { - amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; - return; - } - - if (amd_iommu_snp_en && - !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { - pr_warn("Force to disable Virtual APIC due to SNP\n"); - amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; - return; - } - /* Enabling GAM and SNPAVIC support */ for_each_iommu(iommu) { - if (iommu_init_ga_log(iommu) || - iommu_ga_log_enable(iommu)) + if (!iommu_ga_log_enable(iommu)) return; iommu_feature_enable(iommu, CONTROL_GAM_EN); From 108657f24706d2209c5ecc5bae933fab3975481e Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Tue, 22 Sep 2026 17:00:35 +0800 Subject: [PATCH 18/69] iommu/amd: Fix leak of data->entry in irq_remapping_alloc() error path In irq_remapping_alloc() each per-IRQ iteration allocates two objects: the amd_ir_data (chip_data) and its data->entry (union irte / struct irte_ga). The out_free_data error path only freed chip_data via kfree(irq_data->chip_data), leaking data->entry for every already initialized IRQ. Free data->entry alongside data on the error path. Signed-off-by: Li RongQing Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index b6838fe6ebc1..c679b621c982 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3925,8 +3925,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, out_free_data: for (i--; i >= 0; i--) { irq_data = irq_domain_get_irq_data(domain, virq + i); - if (irq_data) - kfree(irq_data->chip_data); + if (irq_data && irq_data->chip_data) { + data = irq_data->chip_data; + kfree(data->entry); + kfree(data); + } } for (i = 0; i < nr_irqs; i++) free_irte(iommu, devid, index + i); From 6e87850d1a8f32c1d91cce32c0d92b20f86689ed Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 7 Sep 2026 23:12:06 -0500 Subject: [PATCH 19/69] iommu/amd: Add PerfOpt IOMMU performance optimization support Add support for the AMD IOMMU Performance Optimization (PerfOpt) feature as defined in the AMD I/O Virtualization Technology (IOMMU) Specification, Section 3.4.9 (MMIO Offset 016Ch). This feature allows privileged integrated I/O devices (GPUs) to bypass the IOMMU when directly accessing system memory. The IOMMU only enforces the IR/IW permission bits without GPA->SPA translations. amd_iommu_enable_perfopt() performs a detach/reattach cycle to rehome devices already on the identity domain with ATS/PRI/PASID/GCR3 disabled (skip_caps path). amd_iommu_disable_perfopt() restores those capabilities. The per-device dev_data->perfopt flag tracks state. PERF_OPT_EN is a single control bit per IOMMU, shared by every device behind that IOMMU, while enablement is requested per device. It is therefore reference counted (amd_iommu->perfopt_refcount): armed on the first requesting device and cleared on the last, so one device's teardown never clears the bit while a peer behind the same IOMMU still needs it. The per-device flag is cleared on every teardown path (blocked_domain_attach, release_device, and amd_iommu_disable_perfopt), dropping the reference with it, so a reused dev_data never carries stale PerfOpt state onto its next bind. On suspend/resume the hardware is reprogrammed from scratch: amd_iommu_perfopt_clear() forces the bit off without touching the reference count, and amd_iommu_perfopt_restore() re-asserts it from the count after early_enable_iommu(), so armed devices keep the optimization across resume without relying on each consumer driver to re-arm. The exported amd_iommu_enable_perfopt()/amd_iommu_disable_perfopt() run only from a consumer driver's bind/unbind path. group->mutex is not exposed to drivers, but a device bound to its native driver cannot have its IOMMU domain changed concurrently by the core, which serializes the detach/attach pair against core-driven attach. PerfOpt is opt-in -- only enabled when explicitly requested by a driver. Tested-by: Derek J. Clark Co-developed-by: Jatin Kataria Signed-off-by: Jatin Kataria Signed-off-by: Mario Limonciello Tested-by: Boqun Feng (Netflix) Acked-by: Alex Deucher [joerg.roedel@amd.com: Fixed build failure on i386] Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 3 + drivers/iommu/amd/amd_iommu_types.h | 7 + drivers/iommu/amd/init.c | 43 ++++++ drivers/iommu/amd/iommu.c | 214 ++++++++++++++++++++++++++++ include/linux/amd-iommu.h | 13 +- 5 files changed, 279 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 71113e860859..93845b3c3fbf 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; +int amd_iommu_perfopt_clear(struct amd_iommu *iommu); +int amd_iommu_perfopt_restore(struct amd_iommu *iommu); + /* Protection domain ops */ void amd_iommu_init_identity_domain(void); struct protection_domain *protection_domain_alloc(void); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 8241ef922519..1c1624b7c9a3 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -53,6 +53,7 @@ #define MMIO_MSI_ADDR_LO_OFFSET 0x015C #define MMIO_MSI_ADDR_HI_OFFSET 0x0160 #define MMIO_MSI_DATA_OFFSET 0x0164 +#define MMIO_PERF_OPT_OFFSET 0x016C #define MMIO_INTCAPXT_EVT_OFFSET 0x0170 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180 @@ -87,6 +88,8 @@ #define FEATURE_GLX GENMASK_ULL(15, 14) #define FEATURE_GAM_VAPIC BIT_ULL(21) #define FEATURE_PASMAX GENMASK_ULL(36, 32) +#define FEATURE_PERF_OPT BIT_ULL(45) +#define PERF_OPT_EN BIT(13) #define FEATURE_GIOSUP BIT_ULL(48) #define FEATURE_HASUP BIT_ULL(49) #define FEATURE_EPHSUP BIT_ULL(50) @@ -654,6 +657,9 @@ struct amd_iommu { /* Extended features 2 */ u64 features2; + /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ + int perfopt_refcount; + /* PCI device id of the IOMMU device */ u16 devid; @@ -815,6 +821,7 @@ struct iommu_dev_data { u8 ppr :1; /* Enable device PPR support */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; + bool perfopt; struct ratelimit_state rs; /* Ratelimit IOPF messages */ }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 3c02ebd4ed22..8ec8a6fccaaf 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1943,6 +1943,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, if (!iommu->mmio_base) return -ENOMEM; + if (amd_iommu_perfopt_clear(iommu)) + pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index); + return init_iommu_from_acpi(iommu, h); } @@ -3047,10 +3050,46 @@ static void enable_iommus_vapic(void) #endif } +static int clear_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_clear(iommu); + if (err) + ret = err; + } + + return ret; +} + +static int restore_perfopt_all(void) +{ + struct amd_iommu *iommu; + int err, ret = 0; + + for_each_iommu(iommu) { + err = amd_iommu_perfopt_restore(iommu); + if (err) + ret = err; + } + + return ret; +} + static void disable_iommus(void) { struct amd_iommu *iommu; + /* + * PerfOpt is an optional performance bit, so a failure to clear it must + * not skip the mandatory disable below. This also runs from the void + * amd_iommu_disable() shutdown/kexec path, which cannot report an error. + */ + if (clear_perfopt_all()) + pr_err("Failed to clear PerfOpt while disabling IOMMUs\n"); + for_each_iommu(iommu) iommu_disable(iommu); @@ -3076,6 +3115,10 @@ static void amd_iommu_resume(void *data) for_each_iommu(iommu) early_enable_iommu(iommu); + /* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */ + if (restore_perfopt_all()) + pr_err("Failed to restore PerfOpt after IOMMU resume\n"); + iommu_enable_event_buffer(); amd_iommu_enable_interrupts(); } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index c679b621c982..c9b28e5e582e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2370,6 +2370,9 @@ static int attach_device(struct device *dev, if (ret) goto out; + if (dev_data->perfopt) + goto skip_caps; + /* Setup GCR3 table */ if (pdom_is_sva_capable(domain)) { ret = init_gcr3_table(dev_data, domain); @@ -2394,6 +2397,7 @@ static int attach_device(struct device *dev, pdev_enable_cap_ats(pdev); } +skip_caps: /* Update data structures */ dev_data->domain = domain; spin_lock_irqsave(&domain->lock, flags); @@ -2508,6 +2512,192 @@ static int iommu_init_device_caps(struct iommu_dev_data *dev_data, return 0; } +/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */ +static int __perfopt_write(struct amd_iommu *iommu, bool enable) +{ + u32 old, val, readback; + + if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) + return enable ? -ENODEV : 0; + + old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (old == U32_MAX) + return -EIO; + + val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; + if (val != old) + writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (readback == U32_MAX || + (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) + return -EIO; + return 0; +} + +/* + * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is + * reference counted: armed on the first requesting device, cleared on the last. + */ +static int perfopt_get(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount == 0) { + ret = __perfopt_write(iommu, true); + if (ret) + goto out; + } + iommu->perfopt_refcount++; +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +static int perfopt_put(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret = 0; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0) + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown, + * and suspend). The count is preserved so amd_iommu_perfopt_restore() can + * re-arm on resume. + */ +int amd_iommu_perfopt_clear(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* + * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on + * resume, so devices armed before suspend keep the optimization without each + * consumer driver re-arming. + */ +int amd_iommu_perfopt_restore(struct amd_iommu *iommu) +{ + unsigned long flags; + int ret; + + if (!iommu->mmio_base) + return 0; + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + int ret; + + if (!iommu || !dev_data) + return -ENODEV; + + if (!(iommu->features & FEATURE_PERF_OPT)) + return -ENODEV; + + domain = dev_data->domain; + if (!domain) + return -ENODEV; + + /* Already armed for this device (e.g. re-entry on resume). */ + if (dev_data->perfopt) + return 0; + + /* + * The bit is only architecturally valid while the device is untranslated: + * identity domain with ATS/PRI/PASID off. The identity domain is + * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3 + * table. Re-home the device onto the same identity domain with + * perfopt set, so the attach_device() skip_caps path leaves + * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach + * pattern used by amd_iommu_attach_device(). + * + * Locking: this and amd_iommu_disable_perfopt() run only from the + * consumer driver's bind/unbind path. group->mutex is not exposed to + * drivers, but a device bound to its native driver cannot have its domain + * changed concurrently by the core (VFIO ownership is mutually exclusive; + * sysfs domain changes require an unused group), so the detach/attach pair + * is serialized without it. + */ + dev_data->perfopt = true; + detach_device(&pdev->dev); + ret = attach_device(&pdev->dev, domain); + if (ret) + goto err_restore; + + ret = perfopt_get(iommu); + if (ret) + goto err_rearm; + + dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index); + return 0; + +err_rearm: + detach_device(&pdev->dev); +err_restore: + dev_data->perfopt = false; + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n"); + dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n", + iommu->index, ret); + return ret; +} +EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt); + +void amd_iommu_disable_perfopt(struct pci_dev *pdev) +{ + struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); + struct protection_domain *domain; + + if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain) + return; + + if (WARN_ON(perfopt_put(iommu))) + pci_err(pdev, "failed to clear PerfOpt\n"); + + /* + * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its + * identity domain with the flag cleared, so a later bind without PerfOpt + * sees a normally-capable device. See the locking note in + * amd_iommu_enable_perfopt(). + */ + domain = dev_data->domain; + dev_data->perfopt = false; + detach_device(&pdev->dev); + if (attach_device(&pdev->dev, domain)) + pci_err(pdev, "failed to restore caps after PerfOpt disable\n"); +} +EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt); static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; @@ -2551,6 +2741,14 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) static void amd_iommu_release_device(struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n", + iommu->index); + dev_data->perfopt = false; + } WARN_ON(dev_data->domain); @@ -2925,6 +3123,19 @@ static int blocked_domain_attach_device(struct iommu_domain *domain, struct iommu_domain *old) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + /* + * blocked_domain is also the .release_domain, so this is the normal + * teardown path: drop the reference and clear the flag here too, and + * don't fail teardown if the WARN-guarded write doesn't stick. + */ + if (dev_data->perfopt) { + if (WARN_ON(perfopt_put(iommu))) + dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n", + iommu->index); + dev_data->perfopt = false; + } if (dev_data->domain) detach_device(dev); @@ -2993,6 +3204,9 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; + if (dev_data->perfopt && !pdom_is_in_pt_mode(domain)) + return -EBUSY; + /* * Skip attach device to domain if new domain is same as * devices current domain diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index edcee9f5335a..104817e8d897 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -11,11 +11,11 @@ #include struct amd_iommu; +struct pci_dev; #ifdef CONFIG_AMD_IOMMU struct task_struct; -struct pci_dev; extern void amd_iommu_detect(void); @@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif +#ifdef CONFIG_AMD_IOMMU +int amd_iommu_enable_perfopt(struct pci_dev *pdev); +void amd_iommu_disable_perfopt(struct pci_dev *pdev); +#else +static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + return 0; +} +static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } +#endif + #endif /* _ASM_X86_AMD_IOMMU_H */ From e74db71a3530740bb783e3a01420f6af245630b4 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 7 Sep 2026 23:12:07 -0500 Subject: [PATCH 20/69] drm/amdgpu: Enable PerfOpt IOMMU perf optimization when GPU in identity domain Enable PerfOpt via amd_iommu_enable_perfopt() when the GPU's iommu_perfopt module parameter is enabled (default 1) and the GPU resides in the identity domain. The identity domain means the GPU is already performing direct DMA with the IOMMU only enforcing IR/IW permission bits -- no GPA->SPA translations. amd_iommu_enable_perfopt() clears ATS, PRI, PASID and SVA for the device. This is safe in identity domain because DTE[I]=0 means the IOMMU already returns target abort for ATS requests from this peripheral and the GPU manages its own TLB. PerfOpt is a soft, optional latency optimization: failing to arm it (for example on an IOMMU that does not implement the feature, which returns -ENODEV) must not be fatal, so probe and resume warn and continue rather than aborting. PERF_OPT_EN is a per-IOMMU control shared by all devices behind that IOMMU; the IOMMU driver reference counts it so that on systems where multiple devices share one IOMMU, one GPU's teardown does not clear the bit while a peer still requires it. Arming PerfOpt trades IOMMU DMA containment for lower DMA latency. This is enabled by default for GPUs in the identity domain as a deliberate, documented policy and can be disabled with iommu_perfopt=0. The AMD IOMMU spec indicates this is only supported on integrated GPUs so check explicitly for AMD_IS_APU (which is set by amdgpu_device_ip_early_init()). PerfOpt is disabled during GPU init teardown and restored on resume. Tested-by: Derek J. Clark Signed-off-by: Mario Limonciello Tested-by: Boqun Feng (Netflix) Acked-by: Alex Deucher Signed-off-by: Joerg Roedel --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 50 ++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 12 ++++++ 3 files changed, 63 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index a9c6f5d4a639..0e53a02ad1ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -156,6 +156,7 @@ struct amdgpu_watchdog_timer { * Modules parameters. */ extern int amdgpu_modeset; +extern int amdgpu_iommu_perfopt; extern unsigned int amdgpu_vram_limit; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 9269e780feb7..bfe4d90343c4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -3752,6 +3753,28 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) apple_gmux_detect(NULL, NULL))); } +static inline bool amdgpu_device_identity(struct amdgpu_device *adev) +{ + struct pci_dev *pdev = adev->pdev; + struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev); + + if (!domain) + return false; + + return domain->type == IOMMU_DOMAIN_IDENTITY; +} + +static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev) +{ + if (amdgpu_iommu_perfopt == 0) + return false; + + if (!(adev->flags & AMD_IS_APU)) + return false; + + return amdgpu_device_identity(adev); +} + /** * amdgpu_device_init - initialize the driver * @@ -3959,6 +3982,16 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (r) return r; + if (amdgpu_device_use_perfopt(adev)) { + int perfopt_ret = amd_iommu_enable_perfopt(pdev); + + /* Optional optimization; a failure to arm it must not abort probe. */ + if (perfopt_ret) + dev_warn(adev->dev, + "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", + perfopt_ret); + } + /* * No need to remove conflicting FBs for non-display class devices. * This prevents the sysfb from being freed accidently. @@ -4328,6 +4361,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); + if (amdgpu_device_use_perfopt(adev)) + amd_iommu_disable_perfopt(adev->pdev); + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); @@ -4691,6 +4727,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; + if (amdgpu_device_use_perfopt(adev)) { + int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev); + + /* + * Must not return on failure: a bare return would leak the + * SR-IOV VF exclusive-mode acquisition taken above (released + * via the exit: path). + */ + if (perfopt_ret) + dev_warn(adev->dev, + "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", + perfopt_ret); + } + if (adev->in_s0ix) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 5c33c19fd9bc..3ca98091ce48 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -185,6 +185,7 @@ char *amdgpu_disable_cu; char *amdgpu_virtual_display; int amdgpu_enforce_isolation = -1; int amdgpu_modeset = -1; +int amdgpu_iommu_perfopt = -1; /* Specifies the default granularity for SVM, used in buffer * migration and restoration of backing memory when handling @@ -392,6 +393,17 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444); MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)"); module_param_named(aspm, amdgpu_aspm, int, 0444); +/** + * DOC: iommu_perfopt (int) + * Control the AMD IOMMU PerfOpt DMA-latency optimization + * (0 = disable; -1 = enable on supported devices). + * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn) + * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it, + * trading isolation for lower DMA latency. + */ +MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)"); +module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444); + /** * DOC: runpm (int) * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down From d00ac5f7b52bb1f3a2b605f0e773b0397c8e325a Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Mon, 28 Sep 2026 11:27:14 +0800 Subject: [PATCH 21/69] iommu/vt-d: Fix page table level calculation in compute_vasz_lg2_ss() compute_vasz_lg2_ss() finds the optimal Second-Stage page table level by intersecting the maximum guest address width (mgaw) with the hardware's SAGAW capability register. The VT-d spec maps the SAGAW bit field positions as: - Bit 1: 39-bit AGAW (3-level page table, top_level = 2) - Bit 2: 48-bit AGAW (4-level page table, top_level = 3) - Bit 3: 57-bit AGAW (5-level page table, top_level = 4) The fallback paths use bit shifts that are one position too large, causing ffs() to select a deeper page table level than the mgaw window requires: - mgaw > 39: "3 + ffs(sagaw >> 3)" evaluates to top_level = 4 (5-level) instead of top_level = 3 (4-level) when hardware supports both 48-bit (Bit 2) and 57-bit (Bit 3) AGAW. - mgaw > 30: "2 + ffs(sagaw >> 2)" evaluates to top_level = 3 (4-level) instead of top_level = 2 (3-level) when hardware supports both 39-bit (Bit 1) and 48-bit (Bit 2) AGAW. In both cases the selected level is still one that the hardware advertises in its SAGAW capability, so IOVA translation remains functionally correct. However, an unnecessarily deep page table may be selected, adding an extra level of page walk overhead and reducing TLB and cache efficiency without providing any increase in addressable IOVA space beyond what the mgaw window already caps. Fix by decreasing the shift offset by one in each fallback case, ensuring ffs() targets the correct SAGAW bit position and selects the smallest page table level that fully covers the mgaw range: - mgaw > 39: "2 + ffs(sagaw >> 2)" correctly yields top_level = 3 - mgaw > 30: "1 + ffs(sagaw >> 1)" correctly yields top_level = 2 Fixes: d856f9d27885 ("iommupt/vtd: Allow VT-d to have a larger table top than the vasz requires") Signed-off-by: Zhenzhong Duan Reviewed-by: Jason Gunthorpe Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 2e3b3ab216f8..05f351833d0b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2911,10 +2911,10 @@ static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu, *top_level = 4; return min(57, mgaw); } else if (mgaw > 39 && sagaw >= BIT(2)) { - *top_level = 3 + ffs(sagaw >> 3); + *top_level = 2 + ffs(sagaw >> 2); return min(48, mgaw); } else if (mgaw > 30 && sagaw >= BIT(1)) { - *top_level = 2 + ffs(sagaw >> 2); + *top_level = 1 + ffs(sagaw >> 1); return min(39, mgaw); } return 0; From ddea230c9fc28d1f2090ceeab3d66ce2bea658e9 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:15 +0800 Subject: [PATCH 22/69] iommu/vt-d: Avoid out-of-range shift in qi_desc_dev_iotlb_pasid() Callers request a full Device-TLB flush by passing MAX_AGAW_PFN_WIDTH (64 - VTD_PAGE_SHIFT == 52) as @size_order. Two expressions in qi_desc_dev_iotlb_pasid() are not prepared for a value that large: unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1); ... if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order)) The second evaluates to 1UL << 64, which is undefined on all architectures. On x86_64 the shift count masks to zero, IS_ALIGNED(addr, 1) is trivially true, and the alignment sanity check silently degrades into a no-op. The first evaluates to 1UL << 63. That is well defined on 64-bit builds, where unsigned long is 64 bits wide, but is undefined on 32-bit ones. In practice x86 masks the shift count to five bits, so the expression yields 1UL << 31 and ~mask becomes 0x7fffffff. That value is zero-extended when applied to the 64-bit descriptor, so desc->qw1 &= ~mask; clears qw1[63:32] as well as bit 31, collapsing the ADDR field that had just been filled with ones. Reaching that requires a scalable-mode PASID configuration on 32-bit x86, which is not a realistic deployment, but the construct is wrong regardless. Compute the mask with BIT_ULL() so that it is 64-bit on every architecture, and skip the alignment check for a full flush, where it is both meaningless and the source of the out-of-range shift. Note that @size_order must not be clamped below 64 - VTD_PAGE_SHIFT. With S set, hardware derives the invalidation range from the least significant zero bit N of ADDR and matches bits [63:N+1] of the incoming address. For size_order 52 the descriptor sets ADDR[63:12] and clears bit 63, making N = 63 and the comparison range empty, so everything is invalidated. Reducing @size_order to 51 would instead leave N = 62 and cause hardware to compare address bit 63; since callers pass an address of 0, only the lower half of the address space would be invalidated. Bound @size_order at 64 - VTD_PAGE_SHIFT so that a bogus caller cannot reintroduce an out-of-range shift, without altering the full-flush encoding. The non-PASID variant qi_desc_dev_iotlb() already uses 1ULL and is unaffected. Fixes: f701c9f36bcb7 ("iommu/vt-d: Factor out invalidation descriptor composition") Cc: stable@vger.kernel.org Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260623060122.3796325-1-guanghuifeng%40linux.alibaba.com Assisted-by: Claude:claude-opus-5 Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 23dbe6c24439..5b3d234ae265 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1105,12 +1105,20 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, unsigned int size_order, struct qi_desc *desc) { - unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1); - desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) | QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid); + /* + * The widest range the descriptor can express is a full flush, encoded + * by making bit 63 the least significant zero bit of ADDR, that is + * @size_order == 64 - VTD_PAGE_SHIFT. Bound @size_order there so that + * a caller passing something larger cannot produce an out-of-range + * shift below. + */ + if (size_order > 64 - VTD_PAGE_SHIFT) + size_order = 64 - VTD_PAGE_SHIFT; + /* * If S bit is 0, we only flush a single page. If S bit is set, * The least significant zero bit indicates the invalidation address @@ -1120,7 +1128,8 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, * Max Invs Pending (MIP) is set to 0 for now until we have DIT in * ECAP. */ - if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order)) + if (size_order < 64 - VTD_PAGE_SHIFT && + !IS_ALIGNED(addr, BIT_ULL(VTD_PAGE_SHIFT + size_order))) pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n", addr, size_order); @@ -1136,7 +1145,7 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1, VTD_PAGE_SHIFT); /* Clear size_order bit to indicate size */ - desc->qw1 &= ~mask; + desc->qw1 &= ~BIT_ULL(VTD_PAGE_SHIFT + size_order - 1); /* Set the S bit to indicate flushing more than 1 page */ desc->qw1 |= QI_DEV_EIOTLB_SIZE; } From 53e15f28829ff304f94883e8f3f1f5074aa2a0a2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:16 +0800 Subject: [PATCH 23/69] iommu/vt-d: Do not ignore context table copy failures copy_translation_tables() currently logs copy_context_table() failures but still returns success, so partial copy failures are silently ignored. That means Intel IOMMU may run with only part of the old tables copied. Then some old domain IDs may not be reserved, and later may be reused by new domains. With stale hardware cache entries still around, this can cause bad DMA translations, DMA faults, or domain aliasing. Fix by aborting on the first context-table copy failure, freeing temporary context-table pages, and returning an error so caller falls back to a clean root table path. Fixes: f93b4ac5929a ("iommu/vt-d: Use ida to manage domain id") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 05f351833d0b..bf8b3e3edf3b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1591,7 +1591,7 @@ static int copy_translation_tables(struct intel_iommu *iommu) if (ret) { pr_err("%s: Failed to copy context table for bus %d\n", iommu->name, bus); - continue; + goto err_free_ctxt_tbls; } } @@ -1623,11 +1623,27 @@ static int copy_translation_tables(struct intel_iommu *iommu) memunmap(old_rt); return 0; +err_free_ctxt_tbls: + /* + * None of these tables have been linked into iommu->root_entry yet, + * so they are unreachable and must be freed here. + */ + for (bus = 0; bus < ctxt_table_entries; bus++) + iommu_free_pages(ctxt_tbls[bus]); + kfree(ctxt_tbls); out_unmap: memunmap(old_rt); err_free_bitmap: bitmap_free(iommu->copied_tables); iommu->copied_tables = NULL; + + /* + * Only reservations taken from the old context entries can be in the + * ida at this point; no domain has been allocated on this IOMMU yet. + * ida_destroy() empties it and leaves it ready for reuse. + */ + ida_destroy(&iommu->domain_ida); + return ret; } From 12a2ea2d91bea9b1f8f0a6608433210d0abd1291 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:17 +0800 Subject: [PATCH 24/69] iommu/vt-d: Handle DID reservation errors when copying context tables When kdump reuses old translation tables, all old domain IDs (DIDs) must be reserved in the new kernel before new domains are created. Today copy_context_table() ignores ida_alloc_range() return values, so a real -ENOMEM can be missed and an ID may stay unreserved. That can allow DID reuse while stale hardware cache entries still exist, risking domain aliasing. Fix this by moving DID reservation into a helper that: - treats duplicate reservations (-ENOSPC) as expected success, - skips out-of-range IDs as success, - propagates real allocation failures (like -ENOMEM), and - normalize successful return values. On failure, unwind as in existing copy-allocation failure paths. Fixes: f93b4ac5929a ("iommu/vt-d: Use ida to manage domain id") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 39 +++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index bf8b3e3edf3b..85400d0ab334 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1452,12 +1452,39 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) } } +/* + * Reserve a domain ID inherited from the previous kernel so that it is not + * handed out again while the copied translation structures are still live. + * + * Returns 0 when the ID is reserved, was already reserved, or cannot be + * re-assigned, and a negative errno for a genuine allocation failure. + */ +static int reserve_domain_id(struct intel_iommu *iommu, int did) +{ + int ret; + + if (did < 0 || did >= iommu->max_domain_id) + return 0; + + ret = ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL); + /* + * Devices sharing a domain share its ID, so the same ID is seen in + * more than one context entry; -ENOSPC merely reports that it is + * already reserved. On success the allocated ID is returned, which + * is not an error either. + */ + if (ret == -ENOSPC || ret >= 0) + return 0; + + return ret; +} + static int copy_context_table(struct intel_iommu *iommu, struct root_entry *old_re, struct context_entry **tbl, int bus, bool ext) { - int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did; + int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0; struct context_entry *new_ce = NULL, ce; struct context_entry *old_ce = NULL; struct root_entry re; @@ -1520,9 +1547,13 @@ static int copy_context_table(struct intel_iommu *iommu, if (!context_present(&ce)) continue; - did = context_domain_id(&ce); - if (did >= 0 && did < iommu->max_domain_id) - ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL); + ret = reserve_domain_id(iommu, context_domain_id(&ce)); + if (ret) { + /* Not yet published through @tbl, so free it here. */ + iommu_free_pages(new_ce); + new_ce = NULL; + goto out_unmap; + } set_context_copied(iommu, bus, devfn); new_ce[idx] = ce; From 07a64cbe15c4b0bf1d55208f3e945ba8a53075a0 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:18 +0800 Subject: [PATCH 25/69] iommu/vt-d: Reserve scalable-mode DIDs from PASID entries during copy copy_context_table() reserves old domain IDs (DIDs) from context entries. That works for legacy mode, but not for scalable mode: scalable context entries do not carry DID, so this path ends up reserving the wrong value (often DID 0) repeatedly. In scalable mode, real DIDs are stored in PASID table entries. If they are not reserved during kdump table copy, new domains may reuse old DIDs while stale PASID/IOTLB cache state still exists, causing translation conflicts and DMA faults. Fix this by walking PASID tables in scalable mode, and reserving DIDs from present PASID entries. If PASID structures cannot be remapped, return error so caller can fall back safely instead of continuing with an unsafe DID space. Fixes: 0c5f6c0d8201 ("iommu/vt-d: Fix kdump kernels boot failure with scalable mode") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 74 ++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 85400d0ab334..c9e246e8f8e2 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1479,6 +1479,70 @@ static int reserve_domain_id(struct intel_iommu *iommu, int did) return ret; } +/* + * Reserve the domain IDs used by a scalable mode context entry copied from + * the previous kernel. + */ +static int copy_pasid_table_dids(struct intel_iommu *iommu, struct context_entry *ce) +{ + struct pasid_dir_entry *dir; + unsigned long dir_size; + phys_addr_t dir_phys; + int ret = 0; + int i, j; + + dir_phys = ce->lo & VTD_PAGE_MASK; + if (!dir_phys) + return 0; + + dir_size = get_pasid_dir_size(ce); + dir = memremap(dir_phys, dir_size * sizeof(*dir), MEMREMAP_WB); + if (!dir) + return -ENOMEM; + + for (i = 0; i < dir_size; i++) { + struct pasid_entry *table; + phys_addr_t table_phys; + + if (!pasid_pde_is_present(&dir[i])) + continue; + + /* + * Do not use get_pasid_table_from_pde(); that returns a + * phys_to_virt() pointer, which is not valid for memory + * owned by the previous kernel. + */ + table_phys = READ_ONCE(dir[i].val) & PDE_PFN_MASK; + if (!table_phys) + continue; + + /* A PASID table is one page: PASID_TBL_ENTRIES * 64 bytes. */ + table = memremap(table_phys, PAGE_SIZE, MEMREMAP_WB); + if (!table) { + ret = -ENOMEM; + goto out; + } + + for (j = 0; j < PASID_TBL_ENTRIES; j++) { + if (!pasid_pte_is_present(&table[j])) + continue; + + ret = reserve_domain_id(iommu, pasid_get_domain_id(&table[j])); + if (ret) { + memunmap(table); + goto out; + } + } + + memunmap(table); + } + +out: + memunmap(dir); + + return ret; +} + static int copy_context_table(struct intel_iommu *iommu, struct root_entry *old_re, struct context_entry **tbl, @@ -1546,8 +1610,14 @@ static int copy_context_table(struct intel_iommu *iommu, if (!context_present(&ce)) continue; - - ret = reserve_domain_id(iommu, context_domain_id(&ce)); + /* + * The context entry only holds a domain ID in legacy mode. + * In scalable mode the IDs are in the PASID table entries. + */ + if (ext) + ret = copy_pasid_table_dids(iommu, &ce); + else + ret = reserve_domain_id(iommu, context_domain_id(&ce)); if (ret) { /* Not yet published through @tbl, so free it here. */ iommu_free_pages(new_ce); From 91b55eb171cb72c9d0bb8c5bb1dfb32606a2cca7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:19 +0800 Subject: [PATCH 26/69] iommu/vt-d: Use old domain parameter when attaching the blocking domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit blocking_domain_attach_dev() drops the old domain’s iopf reference using info->domain, but that value may already be cleared by device_block_translation(). On attach failure fallback paths, this can cause the function to drop a NULL-domain ref instead of @old, leaking the real old-domain reference. Repeated leaks grow info->iopf_refcount, keep the device stuck on the iopf queue, and can trigger WARN_ON(info->iopf_refcount) when PRI is disabled. Use the core-provided @old parameter directly. It always identifies the correct domain to release and matches other attach paths. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index c9e246e8f8e2..1fb9b80f3b5e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2899,9 +2899,7 @@ static int blocking_domain_attach_dev(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { - struct device_domain_info *info = dev_iommu_priv_get(dev); - - iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev); + iopf_for_domain_remove(old, dev); device_block_translation(dev); return 0; } From 83871c3ae10e3dce95408175df94ecfa0280e15f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:20 +0800 Subject: [PATCH 27/69] iommu/vt-d: Fix iopf refcount leak in nested attach intel_nested_attach_dev() takes an iopf reference for the new domain but does not drop the possible reference from the old domain. This leaks info->iopf_refcount, can keep the device permanently on the iopf queue, and later triggers WARN_ON(info->iopf_refcount) when PRI is disabled. Fix this by dropping the possible reference from the old domain after the nested translation setup completes. Fixes: 17fce9d2336d9 ("iommu/vt-d: Put iopf enablement in domain attach path") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/nested.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 2b979bec56ce..f84fc8b41fde 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -59,6 +59,8 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, if (ret) goto disable_iopf; + iopf_for_domain_remove(old, dev); + info->domain = dmar_domain; info->domain_attached = true; spin_lock_irqsave(&dmar_domain->lock, flags); From 5fbb0fa7327667ade8fa61921425bd4f8803e272 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 28 Sep 2026 11:27:21 +0800 Subject: [PATCH 28/69] iommu/vt-d: Drop old iopf ref only after attach succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit identity_domain_attach_dev() currently removes the old domain’s iopf reference before programming pass-through. If pass-through setup fails, attach fails but the old domain is still effectively attached — now with its IOPF ref already dropped. This can undercount info->iopf_refcount and may disable iopf queue handling while the old domain can still issue page requests. Fix it by removing the old domain’s iopf reference only after pass-through setup succeeds, matching other attach paths. Fixes: 236dd58fabd2 ("iommu/vt-d: Fix iopf_refcount leak on RID domain replacement") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 1fb9b80f3b5e..e3ec7f3b7826 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3985,6 +3985,14 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, if (dev_is_real_dma_subdevice(dev)) return 0; + if (sm_supported(iommu)) + ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); + else + ret = device_setup_pass_through(dev); + + if (ret) + return ret; + /* * The identity domain has no iopf_handler, so no IOPF reference is * taken for it. The reference held by the old domain must still be @@ -3992,16 +4000,9 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, * not affect the IOPF reference count. */ iopf_for_domain_remove(old, dev); + info->domain_attached = true; - if (sm_supported(iommu)) - ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); - else - ret = device_setup_pass_through(dev); - - if (!ret) - info->domain_attached = true; - - return ret; + return 0; } static int identity_domain_set_dev_pasid(struct iommu_domain *domain, From 754d07b940c4ce025fab022b548fb5c3d11bf766 Mon Sep 17 00:00:00 2001 From: Guanghui Feng Date: Mon, 28 Sep 2026 11:27:22 +0800 Subject: [PATCH 29/69] iommu/vt-d: Fix IQE handling to cover all descriptors in submission range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an Invalidation Queue Error (IQE) occurs, hardware halts fetching and IQH points at the faulting descriptor. The previous code only checked whether IQH matched the first descriptor index of the current submission, missing faults on any other descriptor within the batch. Expand the check to cover the entire submission range [index, wait_index], accounting for circular wrap-around. Furthermore, after detecting IQE, the old recovery only replaced the single faulting slot with a copy of the wait descriptor and immediately returned -EINVAL. This left two problems: a) Hardware resumed fetching and could hit another invalid descriptor in the same abandoned batch, raising a second IQE that no submitter would claim — permanently deadlocking the queue. b) The caller reclaimed all batch slots (QI_FREE) while hardware might still be asynchronously processing descriptors from that batch, allowing concurrent overwrite and descriptor corruption. Fix both by introducing qi_drain_remaining_descs(): upon IQE detection, overwrite the stranded slots in [IQH, wait_index) with fenced no-op wait descriptors, resubmit the wait descriptor at wait_index, clear IQE, and spin until hardware signals QI_DONE (with DMAR_OPERATION_TIMEOUT). This guarantees hardware has fully drained the batch before the caller reclaims slots. Fixes: 8a1d82462540 ("iommu/vt-d: Multiple descriptors per qi_submit_sync()") Signed-off-by: Guanghui Feng Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 70 ++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index ba675b08cd20..6377ba97a9ea 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1344,6 +1344,49 @@ static void qi_dump_fault(struct intel_iommu *iommu, u32 fault) (unsigned long long)desc->qw1); } +static int qi_drain_remaining_descs(struct intel_iommu *iommu, int head, + int wait_index) +{ + struct q_inval *qi = iommu->qi; + int shift = qi_shift(iommu); + struct qi_desc wait_desc = {}; + struct qi_desc nop_desc = {}; + cycles_t start; + + nop_desc.qw0 = QI_IWD_FENCE | QI_IWD_TYPE; + + wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) | QI_IWD_PRQ_DRAIN | + QI_IWD_STATUS_WRITE | QI_IWD_FENCE | QI_IWD_TYPE; + wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]); + + while (head != wait_index) { + memcpy(qi->desc + (head << shift), &nop_desc, 1 << shift); + head = (head + 1) % QI_LENGTH; + } + + WRITE_ONCE(qi->desc_status[wait_index], QI_IN_USE); + memcpy(qi->desc + (wait_index << shift), &wait_desc, 1 << shift); + + /* + * Order the descriptor rewrites before the writel() that restarts + * the fetch engine. + */ + wmb(); + + writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); + + start = get_cycles(); + while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) { + if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start)) { + pr_err("Timeout draining invalidation queue after IQE\n"); + return -ETIMEDOUT; + } + cpu_relax(); + } + + return 0; +} + static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) { u32 fault; @@ -1366,18 +1409,25 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) * is cleared. */ if (fault & DMA_FSTS_IQE) { + int head_idx, ret; + head = readl(iommu->reg + DMAR_IQH_REG); - if ((head >> shift) == index) { - struct qi_desc *desc = qi->desc + head; + head_idx = (head >> shift) % QI_LENGTH; + + /* + * The faulting descriptor can be anywhere within the current + * submission's range [index, wait_index]. Since the queue is + * circular, this submission may wrap around QI_LENGTH + * (index > wait_index in that case), so check both the + * non-wrapped and wrapped cases of the range. + */ + if (index <= wait_index ? + (head_idx >= index && head_idx <= wait_index) : + (head_idx >= index || head_idx <= wait_index)) { + ret = qi_drain_remaining_descs(iommu, head_idx, wait_index); + if (ret) + return ret; - /* - * desc->qw2 and desc->qw3 are either reserved or - * used by software as private data. We won't print - * out these two qw's for security consideration. - */ - memcpy(desc, qi->desc + (wait_index << shift), - 1 << shift); - writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); pr_info("Invalidation Queue Error (IQE) cleared\n"); return -EINVAL; } From 20a83ff21712790786c087dab7315fdfa34035ee Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Sun, 27 Sep 2026 14:09:00 +0100 Subject: [PATCH 30/69] iommu_pt: Fix test_pgsize_boundary() failure on narrow-OA formats test_pgsize_boundary() tests boundary mapping with a hardcoded physical address that requires 38 bits of physical address space. However, the upcoming page table format for the Broadcom BCM2712 IOMMU only supports 37 bits of output address space. This causes a test failure as iommu_map() rejects the large address with -ERANGE. Drop the high bits of the address (they are not relevant to the boundary calculations being tested) so that it fits into 32 bits. Reviewed-by: Jason Gunthorpe Signed-off-by: Daniel Drake Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/kunit_iommu_pt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h index ece1c9b8c55d..9fc71db74234 100644 --- a/drivers/iommu/generic_pt/kunit_iommu_pt.h +++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h @@ -403,7 +403,7 @@ static void test_pgsize_boundary(struct kunit *test) priv->smallest_pgsz != SZ_4K) kunit_skip(test, "Format does not have the required range"); - do_map(test, 0xfef80000, 0x208b95d000, 0xfef9ffff - 0xfef80000 + 1); + do_map(test, 0xfef80000, 0x8b95d000, 0xfef9ffff - 0xfef80000 + 1); } /* See https://lore.kernel.org/r/20250826143816.38686-1-eugkoira@amazon.com */ From c5c30144bbd57dad377e43b27de466ca13226c88 Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Sun, 27 Sep 2026 14:09:01 +0100 Subject: [PATCH 31/69] dt-bindings: iommu: Add Broadcom BCM2712 IOMMU Add bindings for the Broadcom BCM2712 IOMMUs and their shared TLB cache. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Daniel Drake Signed-off-by: Joerg Roedel --- .../bindings/iommu/brcm,bcm2712-iommu.yaml | 54 +++++++++++++++++++ .../bindings/iommu/brcm,bcm2712-iommuc.yaml | 40 ++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml create mode 100644 Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml new file mode 100644 index 000000000000..5ca8ce469ec3 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom BCM2712 IOMMU + +maintainers: + - Daniel Drake + - Florian Fainelli + - Jim Quinlan + +description: + The BCM2712 SoC features multiple independent IOMMU instances providing + address translation for multimedia hardware blocks, such as the VC6 display + pipeline, camera receivers, and ISP. Each IOMMU has its own page tables, + control registers, and local TLB, and is additionally linked to a centralized + L2 TLB (IOMMUC). + +properties: + compatible: + const: brcm,bcm2712-iommu + + reg: + maxItems: 1 + + '#iommu-cells': + const: 0 + + brcm,iommu-cache: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the shared IOMMU cache (IOMMUC). Each IOMMU instance caches + translated descriptors in this centralized L2 TLB cache block and must + trigger cache invalidation commands on it when updating or unmapping page + table entries. + +required: + - compatible + - reg + - '#iommu-cells' + - brcm,iommu-cache + +additionalProperties: false + +examples: + - | + iommu@5200 { + compatible = "brcm,bcm2712-iommu"; + reg = <0x5200 0x80>; + brcm,iommu-cache = <&iommuc>; + #iommu-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml new file mode 100644 index 000000000000..47c584ad667a --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommuc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom BCM2712 Shared IOMMU Cache (IOMMUC) + +maintainers: + - Daniel Drake + - Florian Fainelli + - Jim Quinlan + +description: + The BCM2712 IOMMUC is a centralized Translation Lookaside Buffer (TLB) cache + which accelerates address translation across the SoC's IOMMU devices. If an + address mapping is not found in the IOMMU's local TLB cache, then this + IOMMUC is consulted as a L2 TLB. The SoC includes a single IOMMUC which is + shared between all IOMMUs, and it must be explicitly invalidated when + modifying or unmapping IOMMU page tables. + +properties: + compatible: + const: brcm,bcm2712-iommuc + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + iommuc@5b00 { + compatible = "brcm,bcm2712-iommuc"; + reg = <0x5b00 0x80>; + }; From b445c70575ad94adb3bfdbde48c3e2c1c61d47da Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Sun, 27 Sep 2026 14:09:02 +0100 Subject: [PATCH 32/69] dt-bindings: display: brcm,bcm2835-hvs: Document iommus property On BCM2712, the Hardware Video Scaler (HVS) is connected behind an IOMMU. Add the optional iommus property to allow describing this connection. Acked-by: Conor Dooley Reviewed-by: Florian Fainelli Signed-off-by: Daniel Drake Signed-off-by: Joerg Roedel --- .../devicetree/bindings/display/brcm,bcm2835-hvs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml index 9aca38a58a16..8f8aae9f3788 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml @@ -35,6 +35,9 @@ properties: minItems: 1 maxItems: 2 + iommus: + maxItems: 1 + required: - compatible - reg From 3e69ac737ed4ae248c49f8a70faa0a96eeecd59a Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Sun, 27 Sep 2026 14:09:03 +0100 Subject: [PATCH 33/69] iommu/generic_pt: Add Broadcom BCM2712 page table format Implement generic_pt support for the 2-level page table format used by the BCM2712 IOMMUs. Adapted from Raspberry Pi's downstream bcm2712-iommu driver (original author Nick Hollinghurst). Reviewed-by: Jason Gunthorpe Signed-off-by: Daniel Drake Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/.kunitconfig | 1 + drivers/iommu/generic_pt/Kconfig | 10 + drivers/iommu/generic_pt/fmt/Makefile | 2 + drivers/iommu/generic_pt/fmt/bcm2712.h | 288 +++++++++++++++++++ drivers/iommu/generic_pt/fmt/defs_bcm2712.h | 18 ++ drivers/iommu/generic_pt/fmt/iommu_bcm2712.c | 6 + include/linux/generic_pt/common.h | 6 + include/linux/generic_pt/iommu.h | 12 + 8 files changed, 343 insertions(+) create mode 100644 drivers/iommu/generic_pt/fmt/bcm2712.h create mode 100644 drivers/iommu/generic_pt/fmt/defs_bcm2712.h create mode 100644 drivers/iommu/generic_pt/fmt/iommu_bcm2712.c diff --git a/drivers/iommu/generic_pt/.kunitconfig b/drivers/iommu/generic_pt/.kunitconfig index 0bb98fe581fe..a3da74328f12 100644 --- a/drivers/iommu/generic_pt/.kunitconfig +++ b/drivers/iommu/generic_pt/.kunitconfig @@ -7,6 +7,7 @@ CONFIG_IOMMU_PT_AMDV1=y CONFIG_IOMMU_PT_VTDSS=y CONFIG_IOMMU_PT_RISCV64=y CONFIG_IOMMU_PT_X86_64=y +CONFIG_IOMMU_PT_BCM2712=y CONFIG_IOMMU_PT_KUNIT_TEST=y CONFIG_IOMMUFD=y diff --git a/drivers/iommu/generic_pt/Kconfig b/drivers/iommu/generic_pt/Kconfig index f4ed1add58b7..2a81dac960ac 100644 --- a/drivers/iommu/generic_pt/Kconfig +++ b/drivers/iommu/generic_pt/Kconfig @@ -72,6 +72,15 @@ config IOMMU_PT_X86_64 Selected automatically by an IOMMU driver that uses this format. +config IOMMU_PT_BCM2712 + tristate "IOMMU page table for Broadcom BCM2712" + help + iommu_domain implementation for the Broadcom BCM2712 IOMMU found on + Raspberry Pi 5. It supports 4K page sizes over a 2-level page table + format. + + Selected automatically by an IOMMU driver that uses this format. + config IOMMU_PT_KUNIT_TEST tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS depends on KUNIT @@ -79,6 +88,7 @@ config IOMMU_PT_KUNIT_TEST depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64 depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64 depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS + depends on IOMMU_PT_BCM2712 || !IOMMU_PT_BCM2712 default KUNIT_ALL_TESTS help Enable kunit tests for GENERIC_PT and IOMMU_PT that covers all the diff --git a/drivers/iommu/generic_pt/fmt/Makefile b/drivers/iommu/generic_pt/fmt/Makefile index ea024d582594..82ca3c36823d 100644 --- a/drivers/iommu/generic_pt/fmt/Makefile +++ b/drivers/iommu/generic_pt/fmt/Makefile @@ -9,6 +9,8 @@ iommu_pt_fmt-$(CONFIG_IOMMU_PT_RISCV64) += riscv64 iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64 +iommu_pt_fmt-$(CONFIG_IOMMU_PT_BCM2712) += bcm2712 + IOMMU_PT_KUNIT_TEST := define create_format obj-$(2) += iommu_$(1).o diff --git a/drivers/iommu/generic_pt/fmt/bcm2712.h b/drivers/iommu/generic_pt/fmt/bcm2712.h new file mode 100644 index 000000000000..2c5b034441c1 --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/bcm2712.h @@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Daniel Drake + * + * BCM2712 IOMMU simple two level page table + */ +#ifndef __GENERIC_PT_FMT_BCM2712_H +#define __GENERIC_PT_FMT_BCM2712_H + +#include "defs_bcm2712.h" +#include "../pt_defs.h" + +#include +#include +#include +#include + +enum { + /* Hardware provides a two-level page table */ + PT_MAX_TOP_LEVEL = 1, + + /* Hardware page size is strictly 4kb */ + PT_GRANULE_LG2SZ = ilog2(SZ_4K), + PT_TABLEMEM_LG2SZ = PT_GRANULE_LG2SZ, + + /* Table entries and leaf entries are 32 bits */ + PT_ITEM_WORD_SIZE = sizeof(pt_bcm2712_entry_t), + + /* Leaf entries encode a 25-bit PFN */ + PT_MAX_OUTPUT_ADDRESS_LG2 = 25 + PT_GRANULE_LG2SZ, + + /* + * For simplicity, only manage mappings within an address space of + * exactly 4GB. This is because our Level 1 directory page will be 4kb + * in the smallest case, permitting 1024 entries pointing at Level 0 + * pages, each permitting 4mb of mapped memory. + */ + PT_MAX_VA_ADDRESS_LG2 = ilog2(SZ_4G), + + /* Level 1 base address is programmed as a 32-bit PFN */ + PT_TOP_PHYS_MASK = GENMASK_ULL(31 + PT_GRANULE_LG2SZ, PT_GRANULE_LG2SZ), +}; + +/* PTE bits */ +enum { + BCM2712PT_VALID = BIT(28), + BCM2712PT_WRITE = BIT(29), + BCM2712PT_PAGESIZE = GENMASK(31, 30), + BCM2712PT_PFN = GENMASK(24, 0), +}; + +#define common_to_bcm2712pt(common_ptr) \ + container_of_const(common_ptr, struct pt_bcm2712, common) +#define to_bcm2712pt(pts) common_to_bcm2712pt((pts)->range->common) + +static inline pt_oaddr_t bcm2712pt_table_pa(const struct pt_state *pts) +{ + return oalog2_mul(FIELD_GET(BCM2712PT_PFN, pts->entry), + PT_GRANULE_LG2SZ); +} +#define pt_table_pa bcm2712pt_table_pa +#define pt_item_oa bcm2712pt_table_pa + +static inline bool bcm2712pt_can_have_leaf(const struct pt_state *pts) +{ + return true; +} +#define pt_can_have_leaf bcm2712pt_can_have_leaf + +/* 4MB pages are installed at level 1, everything else at level 0 */ +static inline unsigned int bcm2712pt_pgsz_lg2_to_level(struct pt_common *common, + unsigned int pgsize_lg2) +{ + return pgsize_lg2 == ilog2(SZ_4M) ? 1 : 0; +} +#define pt_pgsz_lg2_to_level bcm2712pt_pgsz_lg2_to_level + +static inline pt_vaddr_t bcm2712pt_possible_sizes(const struct pt_state *pts) +{ + struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common); + + if (pts->level == 1) + return SZ_4M; + + return SZ_4K | + (table->bigpage_lg2 ? BIT_ULL(table->bigpage_lg2) : 0) | + (table->superpage_lg2 ? BIT_ULL(table->superpage_lg2) : 0); +} +#define pt_possible_sizes bcm2712pt_possible_sizes + +static inline unsigned int +bcm2712pt_entry_num_contig_lg2(const struct pt_state *pts) +{ + struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common); + u32 pgsz = FIELD_GET(BCM2712PT_PAGESIZE, pts->entry); + + if (pts->level != 0 || !(pts->entry & BCM2712PT_VALID)) + return 0; + + /* + * Superpage/bigpage contiguous mapping (really just hinting) is handled + * via PAGESIZE bits on each leaf entry. + */ + if (pgsz == 2 && table->superpage_lg2) + return table->superpage_lg2 - PT_GRANULE_LG2SZ; + else if (pgsz == 1 && table->bigpage_lg2) + return table->bigpage_lg2 - PT_GRANULE_LG2SZ; + return 0; +} +#define pt_entry_num_contig_lg2 bcm2712pt_entry_num_contig_lg2 + +static inline unsigned int bcm2712pt_num_items_lg2(const struct pt_state *pts) +{ + return PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE); +} +#define pt_num_items_lg2 bcm2712pt_num_items_lg2 + +static inline enum pt_entry_type bcm2712pt_load_entry_raw(struct pt_state *pts) +{ + const pt_bcm2712_entry_t *tablep = + pt_cur_table(pts, pt_bcm2712_entry_t); + + pts->entry = READ_ONCE(tablep[pts->index]); + if (!(pts->entry & BCM2712PT_VALID)) + return PT_ENTRY_EMPTY; + + if (pts->level == 1) { + if (FIELD_GET(BCM2712PT_PAGESIZE, pts->entry) == 3) + return PT_ENTRY_OA; + return PT_ENTRY_TABLE; + } + + return PT_ENTRY_OA; +} +#define pt_load_entry_raw bcm2712pt_load_entry_raw + +static inline void +bcm2712pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, + unsigned int oasz_lg2, + const struct pt_write_attrs *attrs) +{ + pt_bcm2712_entry_t *tablep = pt_cur_table(pts, pt_bcm2712_entry_t); + pt_bcm2712_entry_t entry; + + if (!pt_check_install_leaf_args(pts, oa, oasz_lg2)) + return; + + entry = BCM2712PT_VALID | attrs->descriptor_bits | + FIELD_PREP(BCM2712PT_PFN, oalog2_div(oa, PT_GRANULE_LG2SZ)); + + if (pts->level == 1) { + /* Level 1 hugepage (4MB) */ + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 3); + WRITE_ONCE(tablep[pts->index], entry); + pts->entry = entry; + return; + } + + if (oasz_lg2 == PT_GRANULE_LG2SZ) { + WRITE_ONCE(tablep[pts->index], entry); + pts->entry = entry; + } else { + struct pt_bcm2712 *table = + common_to_bcm2712pt(pts->range->common); + u32 *end; + + tablep += pts->index; + end = tablep + log2_to_int(oasz_lg2 - PT_GRANULE_LG2SZ); + + /* + * Leaf entries can contain hints indicating bigpage/superpage + * contiguous mappings to permit TLB optimization + */ + if (oasz_lg2 == table->superpage_lg2) + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 2); + else if (oasz_lg2 == table->bigpage_lg2) + entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 1); + + pts->entry = entry; + for (; tablep != end; tablep++, entry++) + WRITE_ONCE(*tablep, entry); + } +} +#define pt_install_leaf_entry bcm2712pt_install_leaf_entry + +static inline bool bcm2712pt_install_table(struct pt_state *pts, + pt_oaddr_t table_pa, + const struct pt_write_attrs *attrs) +{ + pt_bcm2712_entry_t entry = + BCM2712PT_VALID | + FIELD_PREP(BCM2712PT_PFN, + oalog2_div(table_pa, PT_GRANULE_LG2SZ)); + + return pt_table_install32(pts, entry); +} +#define pt_install_table bcm2712pt_install_table + +static inline void bcm2712pt_attr_from_entry(const struct pt_state *pts, + struct pt_write_attrs *attrs) +{ + attrs->descriptor_bits = pts->entry & BCM2712PT_WRITE; +} +#define pt_attr_from_entry bcm2712pt_attr_from_entry + +static inline unsigned int bcm2712pt_max_sw_bit(struct pt_common *common) +{ + return 2; +} +#define pt_max_sw_bit bcm2712pt_max_sw_bit + +static inline u32 bcm2712pt_sw_bit(unsigned int bitnr) +{ + if (__builtin_constant_p(bitnr) && bitnr > 2) + BUILD_BUG(); + if (PT_WARN_ON(bitnr > 2)) + return 0; + + /* Bits 27, 26 and 25 are believed spare */ + return BIT(27 - bitnr); +} +#define pt_sw_bit bcm2712pt_sw_bit + +/* --- iommu */ +#include +#include + +#define pt_iommu_table pt_iommu_bcm2712 + +/* The common struct is in the per-format common struct */ +static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table) +{ + return &container_of(iommu_table, struct pt_iommu_table, iommu) + ->bcm2712pt.common; +} + +static inline struct pt_iommu *iommu_from_common(struct pt_common *common) +{ + return &container_of(common, struct pt_iommu_table, bcm2712pt.common) + ->iommu; +} + +static inline int bcm2712_pt_iommu_set_prot(struct pt_common *common, + struct pt_write_attrs *attrs, + unsigned int iommu_prot) +{ + attrs->descriptor_bits = 0; + if (iommu_prot & IOMMU_WRITE) + attrs->descriptor_bits |= BCM2712PT_WRITE; + return 0; +} +#define pt_iommu_set_prot bcm2712_pt_iommu_set_prot + +static inline int bcm2712_pt_fmt_init(struct pt_iommu_table *fmt_table, + const struct pt_iommu_bcm2712_cfg *cfg) +{ + fmt_table->bcm2712pt.bigpage_lg2 = cfg->bigpage_lg2; + fmt_table->bcm2712pt.superpage_lg2 = cfg->superpage_lg2; + + pt_top_set_level(&fmt_table->bcm2712pt.common, PT_MAX_TOP_LEVEL); + return 0; +} +#define pt_iommu_fmt_init bcm2712_pt_fmt_init + +static inline void +bcm2712pt_iommu_fmt_hw_info(struct pt_iommu_bcm2712 *table, + const struct pt_range *top_range, + struct pt_iommu_bcm2712_hw_info *info) +{ + info->pt_base = virt_to_phys(top_range->top_table); + PT_WARN_ON(info->pt_base & ~PT_TOP_PHYS_MASK); +} +#define pt_iommu_fmt_hw_info bcm2712pt_iommu_fmt_hw_info + +#if defined(GENERIC_PT_KUNIT) +static const struct pt_iommu_bcm2712_cfg bcm2712_kunit_fmt_cfgs[] = { + [0] = { + .common.hw_max_vasz_lg2 = 32, + .common.hw_max_oasz_lg2 = 37, + .bigpage_lg2 = 16, + .superpage_lg2 = 20, + }, +}; +#define kunit_fmt_cfgs bcm2712_kunit_fmt_cfgs +enum { KUNIT_FMT_FEATURES = 0 }; +#endif + +#endif diff --git a/drivers/iommu/generic_pt/fmt/defs_bcm2712.h b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h new file mode 100644 index 000000000000..f1d983be1716 --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __GENERIC_PT_FMT_DEFS_BCM2712_H +#define __GENERIC_PT_FMT_DEFS_BCM2712_H + +#include +#include + +typedef u32 pt_bcm2712_entry_t; +typedef u64 pt_vaddr_t; +typedef u64 pt_oaddr_t; + +struct bcm2712pt_write_attrs { + pt_bcm2712_entry_t descriptor_bits; + gfp_t gfp; +}; +#define pt_write_attrs bcm2712pt_write_attrs + +#endif diff --git a/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c new file mode 100644 index 000000000000..43fbc3cf3adc --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define PT_FMT bcm2712 +#define PT_SUPPORTED_FEATURES BIT(PT_FEAT_DMA_INCOHERENT) + +#include +#include "iommu_template.h" diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 07ef1c8341a4..1b39b27f0cfa 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -213,4 +213,10 @@ enum { PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START, }; +struct pt_bcm2712 { + struct pt_common common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + #endif diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index dd0edd02a48a..13ebb72a67de 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -346,6 +346,18 @@ struct pt_iommu_x86_64_hw_info { IOMMU_FORMAT(x86_64, x86_64_pt); +struct pt_iommu_bcm2712_cfg { + struct pt_iommu_cfg common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + +struct pt_iommu_bcm2712_hw_info { + phys_addr_t pt_base; +}; + +IOMMU_FORMAT(bcm2712, bcm2712pt); + #undef IOMMU_PROTOTYPES #undef IOMMU_FORMAT #endif From 0bf031ba8c4904d8f871b77d1e419f4a44ab5799 Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Sun, 27 Sep 2026 14:09:04 +0100 Subject: [PATCH 34/69] iommu: Add Broadcom BCM2712 IOMMU driver This IOMMU translates memory access requests for the VC6 display pipeline and various multimedia devices in the Broadcom BCM2712 SoC used on Raspberry Pi 5. The driver uses the generic_pt framework to manage the 2-level page tables. It also provides support for the centralized L2 TLB (IOMMUC), which is shared across the SoC's IOMMU instances and must be flushed when modifying or unmapping page table entries. Adapted from Raspberry Pi's downstream bcm2712-iommu driver (original author Nick Hollinghurst). Reviewed-by: Jason Gunthorpe Signed-off-by: Daniel Drake Signed-off-by: Joerg Roedel --- MAINTAINERS | 12 + drivers/iommu/Kconfig | 15 + drivers/iommu/Makefile | 1 + drivers/iommu/bcm2712-iommu-cache.c | 84 +++++ drivers/iommu/bcm2712-iommu-cache.h | 9 + drivers/iommu/bcm2712-iommu.c | 550 ++++++++++++++++++++++++++++ 6 files changed, 671 insertions(+) create mode 100644 drivers/iommu/bcm2712-iommu-cache.c create mode 100644 drivers/iommu/bcm2712-iommu-cache.h create mode 100644 drivers/iommu/bcm2712-iommu.c diff --git a/MAINTAINERS b/MAINTAINERS index cc3cae2e378b..c5fe736604cf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5166,6 +5166,18 @@ N: bcm2712 N: bcm283* N: raspberrypi +BROADCOM BCM2712 IOMMU DRIVER +M: Daniel Drake +M: Florian Fainelli +M: Jim Quinlan +R: Broadcom internal kernel review list +L: iommu@lists.linux.dev +L: linux-rpi-kernel@lists.infradead.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu* +F: drivers/iommu/bcm2712-iommu* +F: drivers/iommu/generic_pt/fmt/*bcm2712* + BROADCOM BCM281XX/BCM11XXX/BCM216XX ARM ARCHITECTURE M: Florian Fainelli M: Ray Jui diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd69467a..de85fba9c0a7 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -386,6 +386,21 @@ config VSI_IOMMU Say Y here if you want to use this IOMMU in front of these hardware blocks. +config BCM2712_IOMMU + bool "BCM2712 IOMMU driver" + depends on (ARCH_BCM2835 && ARM64) || COMPILE_TEST + select IOMMU_API + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_BCM2712 + help + Support for IOMMU on BCM2712 SoC. This IOMMU can be used by the + display controller and various multimedia devices to perform + efficient memory management. + + Say Y here if you want to use this IOMMU in front of these + hardware blocks. + config IOMMU_DEBUG_PAGEALLOC bool "Debug IOMMU mappings against page allocations" depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 2f05725eaab1..29a26d2a3af1 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -37,4 +37,5 @@ obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o +obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o diff --git a/drivers/iommu/bcm2712-iommu-cache.c b/drivers/iommu/bcm2712-iommu-cache.c new file mode 100644 index 000000000000..8ed436943b83 --- /dev/null +++ b/drivers/iommu/bcm2712-iommu-cache.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * IOMMU driver for BCM2712 TLB cache + * + * Copyright (c) 2023 Raspberry Pi Ltd. + * Copyright (c) 2026 Daniel Drake + * + * The BCM2712 IOMMUC is a centralized TLB which accelerates address translation + * across the SoC's IOMMU devices. If an address mapping is not found in the + * IOMMU's local TLB cache, then this IOMMUC is consulted. The IOMMUC must be + * explicitly invalidated when modifying or unmapping IOMMU page tables. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "bcm2712-iommu-cache.h" + +struct bcm2712_iommu_cache { + struct device *dev; + spinlock_t hw_lock; + void __iomem *reg_base; +}; + +#define MMUC_CONTROL_ENABLE BIT(0) +#define MMUC_CONTROL_FLUSH BIT(1) +#define MMUC_CONTROL_FLUSHING BIT(2) + +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache) +{ + u32 val; + int ret; + + scoped_guard(spinlock_irqsave, &cache->hw_lock) { + writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH, + cache->reg_base); + + ret = readl_poll_timeout_atomic(cache->reg_base, val, + !(val & MMUC_CONTROL_FLUSHING), + 0, 50); + } + + if (ret) + dev_err_ratelimited(cache->dev, "cache flush timed out\n"); +} + +static int bcm2712_iommu_cache_probe(struct platform_device *pdev) +{ + struct bcm2712_iommu_cache *cache; + + cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL); + if (!cache) + return -ENOMEM; + + cache->dev = &pdev->dev; + spin_lock_init(&cache->hw_lock); + + cache->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(cache->reg_base)) + return PTR_ERR(cache->reg_base); + + platform_set_drvdata(pdev, cache); + return 0; +} + +static const struct of_device_id bcm2712_iommu_cache_of_match[] = { + { .compatible = "brcm,bcm2712-iommuc" }, + { /* sentinel */ }, +}; + +static struct platform_driver bcm2712_iommu_cache_driver = { + .probe = bcm2712_iommu_cache_probe, + .driver = { + .name = "bcm2712-iommu-cache", + .of_match_table = bcm2712_iommu_cache_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(bcm2712_iommu_cache_driver); diff --git a/drivers/iommu/bcm2712-iommu-cache.h b/drivers/iommu/bcm2712-iommu-cache.h new file mode 100644 index 000000000000..d2f7851831cc --- /dev/null +++ b/drivers/iommu/bcm2712-iommu-cache.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _BCM2712_IOMMU_CACHE_H +#define _BCM2712_IOMMU_CACHE_H + +struct bcm2712_iommu_cache; + +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache); + +#endif diff --git a/drivers/iommu/bcm2712-iommu.c b/drivers/iommu/bcm2712-iommu.c new file mode 100644 index 000000000000..301855475c88 --- /dev/null +++ b/drivers/iommu/bcm2712-iommu.c @@ -0,0 +1,550 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * IOMMU driver for Broadcom BCM2712 + * + * Copyright (c) 2023-2025 Raspberry Pi Ltd. + * Copyright (c) 2026 Daniel Drake + * + * Each BCM2712 IOMMU has multiple devices hardwired into it, whose + * DMA transactions all route through the IOMMU. There is no stream ID tagging + * or any other kind of segmentation to differentiate between requests from + * different devices. It is also not possible to toggle a specific device + * between iommu-mapped and bypass modes. + * + * The hardware supports the simultaneous configuration of a bypass window + * (where memory accesses operate in identity mode) and a translation aperture + * (supporting page mapping). While combining these two would allow for + * GART-like operation, this driver's paging domain implementation configures + * only the translation aperture (starting at IOVA 0) offering a degree of + * memory protection. + * + * The page table format is a two-level format handled by generic_pt/bcm2712. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bcm2712-iommu-cache.h" +#include "dma-iommu.h" +#include "iommu-pages.h" + +/* BCM2712 IOMMU is organized around 4Kbyte pages */ +#define IOMMU_PAGE_SHIFT 12 +#define IOMMU_PAGE_SIZE (1ul << IOMMU_PAGE_SHIFT) +/* A PTE is 4 bytes */ +#define PTE_SIZE_SHIFT 2 +/* L1/L2 table sizing (IOMMU hardware pages): 1024 entries per page */ +#define PTES_PER_IOPG_SHIFT (IOMMU_PAGE_SHIFT - PTE_SIZE_SHIFT) + +#define MMMU_CTRL_OFFSET 0x00 +#define MMMU_CTRL_CAP_EXCEEDED BIT(27) +#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN BIT(26) +#define MMMU_CTRL_CAP_EXCEEDED_INT_EN BIT(25) +#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN BIT(24) +#define MMMU_CTRL_PT_INVALID BIT(20) +#define MMMU_CTRL_PT_INVALID_ABORT_EN BIT(19) +#define MMMU_CTRL_PT_INVALID_INT_EN BIT(18) +#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN BIT(17) +#define MMMU_CTRL_PT_INVALID_EN BIT(16) +#define MMMU_CTRL_WRITE_VIOLATION BIT(12) +#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN BIT(11) +#define MMMU_CTRL_WRITE_VIOLATION_INT_EN BIT(10) +#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9) +#define MMMU_CTRL_BYPASS BIT(8) +#define MMMU_CTRL_TLB_CLEARING BIT(7) +#define MMMU_CTRL_STATS_CLEAR BIT(3) +#define MMMU_CTRL_TLB_CLEAR BIT(2) +#define MMMU_CTRL_STATS_ENABLE BIT(1) +#define MMMU_CTRL_ENABLE BIT(0) + +#define MMMU_CTRL_OPERATING_FLAGS (\ + MMMU_CTRL_CAP_EXCEEDED_ABORT_EN | \ + MMMU_CTRL_PT_INVALID_ABORT_EN | \ + MMMU_CTRL_PT_INVALID_EN | \ + MMMU_CTRL_WRITE_VIOLATION_ABORT_EN | \ + MMMU_CTRL_STATS_ENABLE | \ + MMMU_CTRL_ENABLE) + +#define MMMU_PT_PA_BASE_OFFSET 0x04 + +#define MMMU_ADDR_CAP_OFFSET 0x14 +#define MMMU_ADDR_CAP_ENABLE BIT(31) +#define ADDR_CAP_SHIFT ilog2(SZ_256M) + +#define MMMU_SHOOT_DOWN_OFFSET 0x18 +#define MMMU_SHOOT_DOWN_SHOOTING BIT(31) +#define MMMU_SHOOT_DOWN_SHOOT BIT(30) + +#define MMMU_BYPASS_START_OFFSET 0x1c +#define MMMU_BYPASS_START_ENABLE BIT(31) + +#define MMMU_BYPASS_END_OFFSET 0x20 +#define MMMU_BYPASS_END_ENABLE BIT(31) + +#define MMMU_MISC_OFFSET 0x24 +#define MMMU_MISC_SINGLE_TABLE BIT(31) + +#define MMMU_ILLEGAL_ADR_OFFSET 0x30 +#define MMMU_ILLEGAL_ADR_ENABLE BIT(31) + +#define MMMU_DEBUG_INFO_OFFSET 0x38 +#define MMMU_DEBUG_INFO_VERSION_MASK 0x0000000Fu +#define MMMU_DEBUG_INFO_VA_WIDTH_MASK 0x000000F0u +#define MMMU_DEBUG_INFO_PA_WIDTH_MASK 0x00000F00u +#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK 0x000FF000u +#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK 0x0FF00000u +#define MMMU_DEBUG_INFO_BYPASS_4M BIT(28) +#define MMMU_DEBUG_INFO_BYPASS BIT(29) + +struct bcm2712_iommu { + struct device *dev; + struct iommu_device iommu; + struct bcm2712_iommu_domain *domain; + struct bcm2712_iommu_cache *cache; + void __iomem *reg_base; + spinlock_t hw_lock; + size_t bigpage_size; + size_t superpage_size; +}; + +struct bcm2712_iommu_domain { + union { + struct iommu_domain base; + struct pt_iommu_bcm2712 pt; + }; + struct bcm2712_iommu *mmu; + void *default_page; +}; + +static inline void bcm2712_iommu_writel(struct bcm2712_iommu *mmu, + unsigned int offset, u32 val) +{ + writel(val, mmu->reg_base + offset); +} + +static inline u32 bcm2712_iommu_readl(struct bcm2712_iommu *mmu, + unsigned int offset) +{ + return readl(mmu->reg_base + offset); +} + +static struct bcm2712_iommu_domain * +to_bcm2712_domain(struct iommu_domain *domain) +{ + return container_of(domain, struct bcm2712_iommu_domain, base); +} + +static void bcm2712_iommu_init(struct bcm2712_iommu *mmu) +{ + unsigned int bigpage_width, superpage_width; + u32 u = bcm2712_iommu_readl(mmu, MMMU_DEBUG_INFO_OFFSET); + u32 pa_width = FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u); + + dev_dbg(mmu->dev, "DEBUG_INFO = 0x%08x\n", u); + WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 || + FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 || + pa_width < 6 || !(u & MMMU_DEBUG_INFO_BYPASS)); + + dma_set_mask_and_coherent(mmu->dev, DMA_BIT_MASK(pa_width + 30u)); + + bigpage_width = FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u); + if (bigpage_width) + mmu->bigpage_size = IOMMU_PAGE_SIZE << bigpage_width; + + superpage_width = FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u); + if (superpage_width) + mmu->superpage_size = IOMMU_PAGE_SIZE << superpage_width; + + /* Disable MMU and clear sticky flags */ + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, + MMMU_CTRL_CAP_EXCEEDED | MMMU_CTRL_PT_INVALID | + MMMU_CTRL_WRITE_VIOLATION | + MMMU_CTRL_STATS_CLEAR); + + /* Put MMU into 2-level mode */ + bcm2712_iommu_writel(mmu, MMMU_MISC_OFFSET, + bcm2712_iommu_readl(mmu, MMMU_MISC_OFFSET) & + ~MMMU_MISC_SINGLE_TABLE); +} + +static int bcm2712_iommu_identity_attach(struct iommu_domain *identity_domain, + struct device *dev, + struct iommu_domain *old) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + + guard(spinlock_irqsave)(&mmu->hw_lock); + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + + return 0; +} + +static struct iommu_domain bcm2712_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &(const struct iommu_domain_ops) { + .attach_dev = bcm2712_iommu_identity_attach, + }, +}; + +static int bcm2712_iommu_clear_and_enable(struct bcm2712_iommu *mmu) + __must_hold(&mmu->hw_lock) +{ + u32 val; + + bcm2712_iommu_cache_flush(mmu->cache); + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, + MMMU_CTRL_OPERATING_FLAGS | MMMU_CTRL_TLB_CLEAR); + return readl_poll_timeout_atomic(mmu->reg_base + MMMU_CTRL_OFFSET, val, + !(val & MMMU_CTRL_TLB_CLEARING), 0, + 50); +} + + +static int bcm2712_iommu_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct pt_iommu_bcm2712_hw_info info; + u32 default_page_pfn; + int ret = 0; + + /* all attached devices must belong to the same IOMMU instance */ + if (mydomain->mmu != mmu) + return -EINVAL; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain == mydomain) + break; + + /* Configure translation aperture */ + bcm2712_iommu_writel(mmu, MMMU_ADDR_CAP_OFFSET, + MMMU_ADDR_CAP_ENABLE | + (domain->geometry.aperture_end >> + ADDR_CAP_SHIFT)); + + /* + * When the IOMMU handles a request, it adds the + * PT_PA_BASE_OFFSET to (IOVA>>32) to calculate the PFN of the + * corresponding L1 directory page. + * IOVA bits [31:22] are then used to fetch the L1 descriptor + * within (which in turn points to the L2 table). + * This clever logic would allow for a L1 table larger than 4kb + * (and hence a larger aperture). + */ + pt_iommu_bcm2712_hw_info(&mydomain->pt, &info); + bcm2712_iommu_writel(mmu, MMMU_PT_PA_BASE_OFFSET, + info.pt_base >> IOMMU_PAGE_SHIFT); + + /* Disable bypass window */ + bcm2712_iommu_writel(mmu, MMMU_BYPASS_START_OFFSET, 0); + bcm2712_iommu_writel(mmu, MMMU_BYPASS_END_OFFSET, 0); + + /* + * A default (error) page is used to catch illegal reads/writes. + */ + default_page_pfn = virt_to_phys(mydomain->default_page) >> + IOMMU_PAGE_SHIFT; + bcm2712_iommu_writel(mmu, MMMU_ILLEGAL_ADR_OFFSET, + MMMU_ILLEGAL_ADR_ENABLE | + default_page_pfn); + + ret = bcm2712_iommu_clear_and_enable(mmu); + if (ret) { + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + break; + } + + mmu->domain = mydomain; + } + + if (ret) + dev_err_ratelimited(mmu->dev, + "TLB clear timed out during attach\n"); + + return ret; +} + +static int bcm2712_iommu_shootdown_range(struct bcm2712_iommu *mmu, + unsigned long iova, size_t size) + __must_hold(&mmu->hw_lock) +{ + unsigned long start_pfn = iova >> IOMMU_PAGE_SHIFT; + unsigned long end_pfn = (iova + size - 1) >> IOMMU_PAGE_SHIFT; + unsigned long pfn; + u32 val; + int ret; + + /* Shootdown register deals with 4 pages at a time */ + for (pfn = round_down(start_pfn, 4); pfn <= end_pfn; pfn += 4) { + bcm2712_iommu_writel(mmu, MMMU_SHOOT_DOWN_OFFSET, + MMMU_SHOOT_DOWN_SHOOT | pfn); + ret = readl_poll_timeout_atomic( + mmu->reg_base + MMMU_SHOOT_DOWN_OFFSET, val, + !(val & MMMU_SHOOT_DOWN_SHOOTING), 0, 50); + if (ret) + return ret; + } + + return 0; +} + +static int bcm2712_iommu_sync_range(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct bcm2712_iommu *mmu = mydomain->mmu; + int ret = 0; + + if (!size) + return 0; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain != mydomain) + return 0; + + /* If invalidating more than 1MB, just do a full TLB clear */ + if (size >= SZ_1M) { + ret = bcm2712_iommu_clear_and_enable(mmu); + } else { + bcm2712_iommu_cache_flush(mmu->cache); + ret = bcm2712_iommu_shootdown_range(mmu, iova, size); + } + } + + if (ret) + dev_err_ratelimited(mmu->dev, + "TLB sync timed out (size=%#zx)\n", size); + + return ret; +} + +static void bcm2712_iommu_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + bcm2712_iommu_sync_range(domain, gather->start, + gather->end - gather->start + 1); + iommu_put_pages_list(&gather->freelist); +} + +static int bcm2712_iommu_sync_map(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + return bcm2712_iommu_sync_range(domain, iova, size); +} + +static void bcm2712_iommu_sync_all(struct iommu_domain *domain) +{ + size_t aperture_size = domain->geometry.aperture_end - + domain->geometry.aperture_start + 1; + + bcm2712_iommu_sync_range(domain, domain->geometry.aperture_start, + aperture_size); +} + +static void bcm2712_iommu_domain_free(struct iommu_domain *domain) +{ + struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain); + struct bcm2712_iommu *mmu = mydomain->mmu; + + scoped_guard(spinlock_irqsave, &mmu->hw_lock) { + if (mmu->domain == mydomain) { + bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0); + mmu->domain = NULL; + } + } + + pt_iommu_deinit(&mydomain->pt.iommu); + if (mydomain->default_page) + iommu_free_pages(mydomain->default_page); + kfree(mydomain); +} + +static const struct iommu_domain_ops bcm2712_paging_domain_ops = { + IOMMU_PT_DOMAIN_OPS(bcm2712), + .attach_dev = bcm2712_iommu_attach_dev, + .iotlb_sync = bcm2712_iommu_sync, + .iotlb_sync_map = bcm2712_iommu_sync_map, + .flush_iotlb_all = bcm2712_iommu_sync_all, + .free = bcm2712_iommu_domain_free, +}; + +static struct iommu_domain *bcm2712_iommu_domain_alloc(struct device *dev) +{ + struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev); + struct bcm2712_iommu_domain *domain; + struct pt_iommu_bcm2712_cfg cfg = {}; + int ret; + + domain = kzalloc_obj(*domain); + if (!domain) + return NULL; + + domain->mmu = mmu; + domain->pt.iommu.iommu_device = mmu->dev; + cfg.common.features = BIT(PT_FEAT_DMA_INCOHERENT); + + /* Bigpage and superpage sizes are typically 64K and 1M, but may vary */ + if (mmu->bigpage_size) + cfg.bigpage_lg2 = ilog2(mmu->bigpage_size); + if (mmu->superpage_size) + cfg.superpage_lg2 = ilog2(mmu->superpage_size); + + /* 2-level format: 10-bit L1 + 10-bit L2 + 12-bit page offset */ + cfg.common.hw_max_vasz_lg2 = + (2 * PTES_PER_IOPG_SHIFT) + IOMMU_PAGE_SHIFT; + + /* PTEs encode a 25-bit output address PFN */ + cfg.common.hw_max_oasz_lg2 = 25 + IOMMU_PAGE_SHIFT; + + ret = pt_iommu_bcm2712_init(&domain->pt, &cfg, GFP_KERNEL); + if (ret) + goto err; + + /* Set up a default (error) page used to catch illegal reads/writes */ + domain->default_page = iommu_alloc_pages_sz(GFP_KERNEL, PAGE_SIZE); + if (!domain->default_page) + goto err; + + domain->base.ops = &bcm2712_paging_domain_ops; + return &domain->base; + +err: + bcm2712_iommu_domain_free(&domain->base); + return NULL; +} + +static struct bcm2712_iommu * +bcm2712_iommu_get_by_fwnode(struct fwnode_handle *fwnode) +{ + struct device *dev __free(put_device) = + bus_find_device_by_fwnode(&platform_bus_type, fwnode); + + return dev ? dev_get_drvdata(dev) : NULL; +} + +static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct bcm2712_iommu *mmu; + + if (!fwspec || !fwspec->iommu_fwnode) + return ERR_PTR(-ENODEV); + + mmu = bcm2712_iommu_get_by_fwnode(fwspec->iommu_fwnode); + if (!mmu) + return ERR_PTR(-ENODEV); + + dev_iommu_priv_set(dev, mmu); + + return &mmu->iommu; +} + +static int bcm2712_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + return iommu_fwspec_add_ids(dev, args->args, 0); +} + +static const struct iommu_ops bcm2712_iommu_ops = { + .identity_domain = &bcm2712_identity_domain, + .domain_alloc_paging = bcm2712_iommu_domain_alloc, + .probe_device = bcm2712_iommu_probe_device, + .device_group = generic_single_device_group, + .get_resv_regions = iommu_dma_get_resv_regions, + .of_xlate = bcm2712_iommu_of_xlate, +}; + +static const struct of_device_id bcm2712_iommu_of_match[] = { + { .compatible = "brcm,bcm2712-iommu" }, + { /* sentinel */ } +}; + +static int bcm2712_iommu_init_cache(struct bcm2712_iommu *mmu, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + struct device_node *cache_np __free(device_node) = + of_parse_phandle(dev->of_node, "brcm,iommu-cache", 0); + if (!cache_np) + return dev_err_probe(dev, -ENOENT, + "missing brcm,iommu-cache property\n"); + + struct platform_device *cache_pdev __free(platform_device_put) = + of_find_device_by_node(cache_np); + if (!cache_pdev) + return dev_err_probe(dev, -EPROBE_DEFER, + "waiting for cache device\n"); + + mmu->cache = platform_get_drvdata(cache_pdev); + if (!mmu->cache) + return dev_err_probe(dev, -EPROBE_DEFER, + "waiting for cache driver probe\n"); + + return 0; +} + +static int bcm2712_iommu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct bcm2712_iommu *mmu; + int ret; + + mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL); + if (!mmu) + return -ENOMEM; + + mmu->dev = dev; + spin_lock_init(&mmu->hw_lock); + + mmu->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mmu->reg_base)) + return PTR_ERR(mmu->reg_base); + + ret = bcm2712_iommu_init_cache(mmu, pdev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mmu); + bcm2712_iommu_init(mmu); + + ret = iommu_device_sysfs_add(&mmu->iommu, dev, NULL, "%s", + dev_name(dev)); + if (ret) + return ret; + + ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, dev); + if (ret) { + iommu_device_sysfs_remove(&mmu->iommu); + return ret; + } + + return 0; +} + +static struct platform_driver bcm2712_iommu_driver = { + .driver = { + .name = "bcm2712-iommu", + .of_match_table = bcm2712_iommu_of_match, + .suppress_bind_attrs = true, + }, + .probe = bcm2712_iommu_probe, +}; +builtin_platform_driver(bcm2712_iommu_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Daniel Drake "); +MODULE_DESCRIPTION("Broadcom BCM2712 IOMMU driver"); From 87098700b3cb953d0a2ccd1946ca01853b22e783 Mon Sep 17 00:00:00 2001 From: Peter Griffin Date: Fri, 11 Sep 2026 21:21:05 +0000 Subject: [PATCH 35/69] dt-bindings: iommu: arm,smmu-v3: Add arm,instdata-override property arm,instdata-override DT property is intended to be used for SMMUs that have a silicon integration issue with the AxPROT[2] signal always being 1 making all transactions appear as instructions. Google Tensor G5 (Laguna) is an example of an SoC with this silicon erratum. The DT property can be specified to enable a software workaround which overrides the incoming attribute to data. Signed-off-by: Peter Griffin Reviewed-by: Rob Herring (Arm) Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/arm,smmu-v3.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml index 82957334bea2..3c044e4f03d4 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml @@ -83,6 +83,16 @@ properties: register access with page 0 offsets. Set for Cavium ThunderX2 silicon that doesn't support SMMU page1 register space. + arm,instdata-override: + type: boolean + description: + Some SoCs like Google Tensor G5 (Laguna) have an SMMU integration + silicon erratum where AxPROT[2] signal is always 1, making all + transactions appear as instructions. + + This property enables a software workaround to override the + instruction/data attribute of incoming traffic to data. + allOf: - if: not: From 937bbba25930d457d95c151cbe7587ae84fdc5a8 Mon Sep 17 00:00:00 2001 From: Peter Griffin Date: Fri, 11 Sep 2026 21:21:06 +0000 Subject: [PATCH 36/69] iommu/arm-smmu-v3: Add support for arm,instdata-override DT property The Google Tensor G5 SoC (Laguna) has a silicon errata with the SMMU integration where the AxPROT[2] signal is always 1, which makes all transactions appear as instructions. To avoid F_PERMISSION faults (due to the VMSAv8-64 requirement that unprivilieged writable buffers are treated as Privileged Execute-Never (PXN)), as a software workaround we make use of the INSTCFG override feature to force all incoming traffic to data. To enable this workaround, the "arm,instdata-override" DT property can be provided in the device tree entry for the SMMU, similar to how existing errata workarounds are handled by this driver. Note this is added for "stage-1" and "stage-2" but we intentionally leave out "bypass" as nothing is downstream of the SMMU that makes use of the Instruction/Data attribute. [peter.griffin: ported to upstream, updated commit message, remove feature flag] Signed-off-by: Pranjal Shrivastava Signed-off-by: Peter Griffin Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 25 +++++++++++++++++---- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 6 +++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 5732f3ba0122..f88647be9447 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -91,6 +91,7 @@ DEFINE_MUTEX(arm_smmu_asid_lock); static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, + { ARM_SMMU_OPT_OVR_INSTCFG_DATA, "arm,instdata-override"}, { 0, NULL}, }; @@ -1271,7 +1272,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | - STRTAB_STE_1_EATS | STRTAB_STE_1_MEV); + STRTAB_STE_1_EATS | STRTAB_STE_1_MEV | + STRTAB_STE_1_INSTCFG); used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); /* @@ -1287,7 +1289,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) if (cfg & BIT(1)) { used_bits[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS | - STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV); + STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV | + STRTAB_STE_1_INSTCFG); used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | @@ -1900,7 +1903,11 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, STRTAB_STE_1_S1STALLD : 0) | FIELD_PREP(STRTAB_STE_1_EATS, - ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_INSTCFG, + smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ? + STRTAB_STE_1_INSTCFG_DATA : + STRTAB_STE_1_INSTCFG_INCOMING)); if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) && s1dss == STRTAB_STE_1_S1DSS_BYPASS) @@ -1952,7 +1959,11 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, target->data[1] = cpu_to_le64( FIELD_PREP(STRTAB_STE_1_EATS, - ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_INSTCFG, + smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ? + STRTAB_STE_1_INSTCFG_DATA : + STRTAB_STE_1_INSTCFG_INCOMING)); if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB) target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB); @@ -5155,6 +5166,12 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (reg & IDR1_ATTR_TYPES_OVR) smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR; + if ((smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA) && + !(reg & IDR1_ATTR_PERMS_OVR)) { + dev_err(smmu->dev, "Inst/Data attribute override not supported\n"); + return -ENXIO; + } + /* Queue sizes, capped to ensure natural alignment */ smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, reg)); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index dd2fee2f560e..efbac8ca6d7a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -52,6 +52,7 @@ struct arm_vsmmu; #define IDR1_QUEUES_PRESET (1 << 29) #define IDR1_REL (1 << 28) #define IDR1_ATTR_TYPES_OVR (1 << 27) +#define IDR1_ATTR_PERMS_OVR (1 << 26) #define IDR1_CMDQS GENMASK(25, 21) #define IDR1_EVTQS GENMASK(20, 16) #define IDR1_PRIQS GENMASK(15, 11) @@ -285,6 +286,10 @@ static inline u32 arm_smmu_strtab_l2_idx(u32 sid) #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) #define STRTAB_STE_1_SHCFG_INCOMING 1UL +#define STRTAB_STE_1_INSTCFG GENMASK_ULL(51, 50) +#define STRTAB_STE_1_INSTCFG_INCOMING 0UL +#define STRTAB_STE_1_INSTCFG_DATA 2UL + #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) @@ -934,6 +939,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_MSIPOLL (1 << 2) #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) +#define ARM_SMMU_OPT_OVR_INSTCFG_DATA (1 << 5) u32 options; struct arm_smmu_cmdq cmdq; From c9b493a312f3f5f92f9bb5613b0ad78eb4d2a1da Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Fri, 25 Sep 2026 15:15:29 +0100 Subject: [PATCH 37/69] iommu/arm-smmu-v3: Add a cmdq_max_n_shift module parameter The command queue depth comes straight from the maximum the hardware advertises in IDR1, which reaches megabytes of coherent DMA per queue. A system with several SMMUv3 instances pays that per instance, and the Tegra241 CMDQV pays it again for every VCMDQ it preallocates. Queue depth only bounds how many commands may be in flight before a sync. A machine driving a handful of devices, or one with a tight memory budget, has no use for the maximum, and no way to say so. Add cmdq_max_n_shift, a cap on the depth given as the log2 of the entry count, the form the hardware itself takes in the LOG2SIZE field of CMDQ_BASE. It defaults to the largest depth the driver allocates, so an unset parameter changes nothing. Decide the depth in arm_smmu_cmdq_max_n_shift(), which applies the parameter to the IDR1 value, so the queue is allocated at the requested size. The Tegra241 CMDQV sizes its VCMDQs from IDR1 itself, so route that through the same helper. Floor the request at one page worth of entries. Without the floor, a small request trips the CMDQ_BATCH_ENTRIES check in arm_smmu_device_hw_probe() and the SMMU fails to probe. The floor also costs nothing: coherent DMA is page granular, so a shallower queue occupies the same memory as one that fills the page. Assisted-by: LLM Reviewed-by: Breno Leitao Reviewed-by: Jason Gunthorpe Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Nicolin Chen Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 40 ++++++++++++++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + .../iommu/arm/arm-smmu-v3/tegra241-cmdqv.c | 2 +- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f88647be9447..23673d6c5b7c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -40,6 +40,11 @@ module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, "Disable MSI-based polling for CMD_SYNC completion."); +static unsigned int cmdq_max_n_shift = CMDQ_MAX_SZ_SHIFT; +module_param(cmdq_max_n_shift, uint, 0444); +MODULE_PARM_DESC(cmdq_max_n_shift, + "Cap on the command queue depth, as log2 of the number of entries. Defaults to the hardware maximum; the queue never shrinks below one page."); + static const struct iommu_ops arm_smmu_ops; static struct iommu_dirty_ops arm_smmu_dirty_ops; @@ -4423,6 +4428,37 @@ static struct iommu_dirty_ops arm_smmu_dirty_ops = { }; /* Probing and initialisation functions */ + +/** + * arm_smmu_queue_max_n_shift() - pick the log2 depth of a queue + * @hw_max_n_shift: log2 depth the hardware advertises in IDR1 + * @ent_sz_shift: log2 of the queue entry size in bytes + * @limit_n_shift: log2 depth to cap the queue at + * + * @limit_n_shift is floored at one page, because coherent DMA is page + * granular: a shallower queue occupies the same memory as one that fills the + * page, and arm_smmu_init_one_queue() stops shrinking at a page too. + */ +static u32 arm_smmu_queue_max_n_shift(u32 hw_max_n_shift, u32 ent_sz_shift, + u32 limit_n_shift) +{ + u32 floor_n_shift = PAGE_SHIFT - ent_sz_shift; + + limit_n_shift = max(limit_n_shift, floor_n_shift); + return min(hw_max_n_shift, limit_n_shift); +} + +/* + * Command queues are also allocated by the Tegra241 CMDQV for its VCMDQs, which + * need the same depth decision. + */ +u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, CMDQ_ENT_SZ_SHIFT, + min(CMDQ_MAX_SZ_SHIFT, cmdq_max_n_shift)); +} + int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, void __iomem *page, unsigned long prod_off, unsigned long cons_off, @@ -5173,8 +5209,8 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) } /* Queue sizes, capped to ensure natural alignment */ - smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_CMDQS, reg)); + smmu->cmdq.q.llq.max_n_shift = + arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, reg)); if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { /* * We don't support splitting up batches, so one batch of diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index efbac8ca6d7a..cbfb3e9f5095 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1171,6 +1171,7 @@ static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain) void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, struct arm_smmu_cmdq *cmdq); +u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift); int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, void __iomem *page, unsigned long prod_off, unsigned long cons_off, diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 6644075c1431..710a4c694b94 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -663,7 +663,7 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq) /* Cap queue size to SMMU's IDR1.CMDQS and ensure natural alignment */ regval = readl_relaxed(smmu->base + ARM_SMMU_IDR1); q->llq.max_n_shift = - min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, regval)); + arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, regval)); /* Use the common helper to init the VCMDQ, and then... */ ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0, From 3f9fa6b9520d69c0fd400b0c260320c0a8a1500c Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Fri, 25 Sep 2026 15:15:30 +0100 Subject: [PATCH 38/69] iommu/arm-smmu-v3: Default queue depths to one page in a kdump kernel All three queues are sized from the maxima the hardware advertises in IDR1 and allocated at probe, up to 4 MB each on a 4K-page kernel. The capture kernel already disables two of them: arm_smmu_device_reset() drops CR0_EVTQEN and CR0_PRIQEN. It still allocates both at full size. A kdump capture kernel runs from a small crashkernel reservation, and every SMMUv3 instance pays that cost again, up to 12 MB apiece. It goes to queues that either serve the handful of devices used to save the dump or are switched off outright, and it is memory the dump itself needs. Size all three queues at one page worth of entries when is_kdump_kernel(). The queues carry commands and fault records rather than DMA data, so dump throughput is unaffected. A shallower command queue only bounds how many commands may be in flight before a sync, which does not matter for the few devices that save the dump. The cmdq_max_n_shift parameter therefore does not apply in a capture kernel. Suggested-by: Kyle McMartin Assisted-by: LLM Reviewed-by: Jason Gunthorpe Tested-by: Yuanhe Shu Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Nicolin Chen Tested-by: Nicolin Chen Reviewed-by: Breno Leitao Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 23673d6c5b7c..a2a4efccc6d9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -43,7 +43,7 @@ MODULE_PARM_DESC(disable_msipolling, static unsigned int cmdq_max_n_shift = CMDQ_MAX_SZ_SHIFT; module_param(cmdq_max_n_shift, uint, 0444); MODULE_PARM_DESC(cmdq_max_n_shift, - "Cap on the command queue depth, as log2 of the number of entries. Defaults to the hardware maximum; the queue never shrinks below one page."); + "Cap on the command queue depth, as log2 of the number of entries. Defaults to the hardware maximum; the queue never shrinks below one page. A kdump kernel always uses one page."); static const struct iommu_ops arm_smmu_ops; static struct iommu_dirty_ops arm_smmu_dirty_ops; @@ -4435,6 +4435,8 @@ static struct iommu_dirty_ops arm_smmu_dirty_ops = { * @ent_sz_shift: log2 of the queue entry size in bytes * @limit_n_shift: log2 depth to cap the queue at * + * A kdump capture kernel gets one page worth of entries whatever the limit. + * * @limit_n_shift is floored at one page, because coherent DMA is page * granular: a shallower queue occupies the same memory as one that fills the * page, and arm_smmu_init_one_queue() stops shrinking at a page too. @@ -4444,10 +4446,27 @@ static u32 arm_smmu_queue_max_n_shift(u32 hw_max_n_shift, u32 ent_sz_shift, { u32 floor_n_shift = PAGE_SHIFT - ent_sz_shift; + if (is_kdump_kernel()) + return min(hw_max_n_shift, floor_n_shift); + limit_n_shift = max(limit_n_shift, floor_n_shift); return min(hw_max_n_shift, limit_n_shift); } +static inline u32 arm_smmu_evtq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, EVTQ_ENT_SZ_SHIFT, + EVTQ_MAX_SZ_SHIFT); +} + +static inline u32 arm_smmu_priq_max_n_shift(u32 hw_max_n_shift) +{ + /* Capped to ensure natural alignment */ + return arm_smmu_queue_max_n_shift(hw_max_n_shift, PRIQ_ENT_SZ_SHIFT, + PRIQ_MAX_SZ_SHIFT); +} + /* * Command queues are also allocated by the Tegra241 CMDQV for its VCMDQs, which * need the same depth decision. @@ -5208,7 +5227,6 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) return -ENXIO; } - /* Queue sizes, capped to ensure natural alignment */ smmu->cmdq.q.llq.max_n_shift = arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, reg)); if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { @@ -5223,10 +5241,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) return -ENXIO; } - smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_EVTQS, reg)); - smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, - FIELD_GET(IDR1_PRIQS, reg)); + smmu->evtq.q.llq.max_n_shift = + arm_smmu_evtq_max_n_shift(FIELD_GET(IDR1_EVTQS, reg)); + smmu->priq.q.llq.max_n_shift = + arm_smmu_priq_max_n_shift(FIELD_GET(IDR1_PRIQS, reg)); /* SID/SSID sizes */ smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); From 991ded6094727570e40392cf3e49383c51d2f948 Mon Sep 17 00:00:00 2001 From: Jingyi Wang Date: Sun, 30 Aug 2026 19:44:03 -0700 Subject: [PATCH 39/69] dt-bindings: arm-smmu: Document GPU SMMU on Qualcomm Maili SoC Document GPU SMMU which uses "hlos" vote clock on Qualcomm Maili SoC. Signed-off-by: Jingyi Wang Reviewed-by: Maria Yu Acked-by: Conor Dooley Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index ffa33b7f4a96..187710fcc567 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -101,6 +101,7 @@ properties: - qcom,glymur-smmu-500 - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 + - qcom,maili-smmu-500 - qcom,milos-smmu-500 - qcom,qcm2290-smmu-500 - qcom,qcs615-smmu-500 @@ -579,6 +580,7 @@ allOf: - qcom,glymur-smmu-500 - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 + - qcom,maili-smmu-500 - qcom,sm8750-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 From 84366fb547ed987c4857311da0e5127d95bf97e6 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 24 Sep 2026 08:56:12 +0000 Subject: [PATCH 40/69] iommu/arm-smmu-v3: Ensure L2 tables are visible before L1 ptrs Commit 6fabce53f6b9 ("iommu/arm-smmu-v3: Add a missing dma_wmb() for hitless STE update") adds a dma_wmb() to arm_smmu_write_entry() to make sure stream tables and context descriptors are observed first. However, STE L1 table descriptors are configured directly via WRITE_ONCE(), where before that they were zeroed with memset() inside dma_direct_alloc() then written to abort in via memset() also in arm_smmu_init_initial_stes() without a barrier in both cases which means that the SMMUv3 can observe the allocated table before the written descriptors causing it to fetch random data. Similarly in arm_smmu_write_cd_l1_desc() where the L1 CD is written after dma_alloc_coherent() with no barriers. Add dma_wmb() in both cases. Fixes: 48ec83bcbcf5 ("iommu/arm-smmu: Add initial driver support for ARM SMMUv3 devices") Reported-by: Sashiko <> Reviewed-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a2a4efccc6d9..658d2cdfa286 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1523,6 +1523,9 @@ static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst, { u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V; + /* Ensure the zero-cleared L2 table is fully visible. */ + dma_wmb(); + /* The HW has 64 bit atomicity with stores to the L2 CD table */ WRITE_ONCE(dst->l2ptr, cpu_to_le64(val)); } @@ -1812,6 +1815,9 @@ static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst, val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1); val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; + /* Ensure the new L2 table is fully visible. */ + dma_wmb(); + /* The HW has 64 bit atomicity with stores to the L2 STE table */ WRITE_ONCE(dst->l2ptr, cpu_to_le64(val)); } From ce58940ccc3d1a377d238d93666c140405f8ce05 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 24 Sep 2026 08:56:13 +0000 Subject: [PATCH 41/69] iommu/arm-smmu-v3-test: Fix arm_smmu_v3_test_debug_print_used_bits() arm_smmu_v3_test_debug_print_used_bits() hardcodes arm_smmu_get_ste_used() instead of using the provided writer->ops->get_used() callback. This caused Context Descriptors to be incorrectly parsed as STEs. Fixes: 56e1a4cc2588 ("iommu/arm-smmu-v3: Add unit tests for arm_smmu_write_entry") Reported-by: Sashiko <> Reviewed-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add671363c82..eae08d4d77ec 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -125,8 +125,8 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, { __le64 used_bits[NUM_ENTRY_QWORDS] = {}; - arm_smmu_get_ste_used(ste, used_bits); - pr_debug("STE used bits: "); + writer->ops->get_used(ste, used_bits); + pr_debug("Entry used bits: "); print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, sizeof(used_bits), false); } From 388986bfc540e98f3f7ca1b6bc8703fc43e279d1 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 24 Sep 2026 08:56:14 +0000 Subject: [PATCH 42/69] iommu/arm-smmu-v3-test: Add missing error checks for inv array arm_smmu_invs_merge() and arm_smmu_invs_alloc() can return NULL or errors which are checked by the driver but not the test. Add KUNIT_ASSERT_NOT_ERR_OR_NULL() after calling them to fail the test instead of accessing NULL or ERR pointers. Fixes: 15a2a5645ad7 ("iommu/arm-smmu-v3: Introduce a per-domain arm_smmu_invs array") Reported-by: Sashiko <> Signed-off-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index eae08d4d77ec..366dcb2b5554 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -704,17 +704,20 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* New array */ test_a = arm_smmu_invs_alloc(0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); /* Test1: merge invs1 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs1); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0, results1[0], results1[1], results1[2]); /* Test2: merge invs2 (new array) */ test_a = arm_smmu_invs_merge(test_b, &invs2); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0, results2[0], results2[1], results2[2]); @@ -726,6 +729,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test4: merge invs3 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs3); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0, results4[0], results4[1], results4[2]); @@ -737,6 +741,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test6: purge test_b (new array) */ test_a = arm_smmu_invs_purge(test_b); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0, results6[0], results6[1], results6[2]); @@ -748,12 +753,14 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test8: merge invs4 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs4); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0, results7[0], results7[1], results7[2]); /* Test9: merge invs5 (new array) */ test_a = arm_smmu_invs_merge(test_b, &invs5); kfree(test_b); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0, results8[0], results8[1], results8[2]); @@ -765,6 +772,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* Test11: purge test_a (new array) */ test_b = arm_smmu_invs_purge(test_a); kfree(test_a); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b); arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0, results10[0], results10[1], results10[2]); From 8105675f33746cce48f5381d02d0356ef79dca89 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 24 Sep 2026 08:56:15 +0000 Subject: [PATCH 43/69] iommu/arm-smmu-v3-test: Fix OOB in arm_smmu_v3_invs_test_verify() arm_smmu_v3_invs_test_verify() validates the array bounds using KUNIT_EXPECT_EQ(), which triggers a failure and continues execution. If invs->num_invs was smaller than expected, the next loop over num_invs would blindly read past the end of the invs->inv[] array. Switch to KUNIT_ASSERT_EQ() to fail and stop the test on bound errors. Fixes: 15a2a5645ad7 ("iommu/arm-smmu-v3: Introduce a per-domain arm_smmu_invs array") Reported-by: Sashiko <> Reviewed-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 366dcb2b5554..244cf34e5a0b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -643,7 +643,7 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, const int *ids, const int *users, const int *ssids) { - KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); + KUNIT_ASSERT_EQ(test, invs->num_invs, num_invs); KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); while (num_invs--) { KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); From b370a5e979f8bb8f34e30ad126ade734d8f128c0 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 24 Sep 2026 08:56:16 +0000 Subject: [PATCH 44/69] iommu/arm-smmu-v3-test: Fix UBSAN error struct arm_smmu_invs marks its flexible array member inv[] with the __counted_by(max_invs). arm_smmu_v3_invs_test() uses invs1 to invs5 which has num_invs = 3 but omit max_invs. This causes the following UBSAN error: [ 2.333240] UBSAN: array-index-out-of-bounds in drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1116:21 [ 2.333604] # arm_smmu_v3_invs_test: lib/ubsan.c:228: array-index-out-of-bounds in drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c [ 2.334129] index 1 is out of range for type 'struct arm_smmu_inv[] __counted_by(max_invs)' (aka 'struct arm_smmu_inv[]') [ 2.335375] CPU: 6 UID: 0 PID: 155 Comm: kunit_try_catch Tainted: G N 7.2.0-g8dcb331fbb72 #2 PREEMPT [ 2.335759] Tainted: [N]=TEST [ 2.335791] Hardware name: linux,dummy-virt (DT) [ 2.336254] Call trace: [ 2.336932] show_stack+0x18/0x24 (C) [ 2.338443] __dump_stack+0x28/0x38 [ 2.338494] dump_stack_lvl+0x54/0x6c [ 2.338518] dump_stack+0x18/0x24 [ 2.338541] ubsan_epilogue+0x10/0x44 [ 2.338565] __ubsan_handle_out_of_bounds+0xb8/0xbc [ 2.338834] arm_smmu_invs_merge+0x688/0x8ac [ 2.338862] arm_smmu_v3_invs_test+0xd4/0x598 [ 2.338890] kunit_try_run_case+0x64/0x160 [ 2.338914] kunit_generic_run_threadfn_adapter+0x28/0x4c [ 2.338955] kthread+0x10c/0x12c [ 2.338984] ret_from_fork+0x10/0x20 [ 2.339433] ---[ end trace ]--- Explicitly set max_invs = 3 on the test arrays to match the number of elements. Fixes: 15a2a5645ad7 ("iommu/arm-smmu-v3: Introduce a per-domain arm_smmu_invs array") Reviewed-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 244cf34e5a0b..e52a7d95c919 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -654,6 +654,7 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, } static struct arm_smmu_invs invs1 = { + .max_invs = 3, .num_invs = 3, .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, @@ -661,6 +662,7 @@ static struct arm_smmu_invs invs1 = { }; static struct arm_smmu_invs invs2 = { + .max_invs = 3, .num_invs = 3, .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ { .type = INV_TYPE_ATS, .id = 4, }, @@ -668,6 +670,7 @@ static struct arm_smmu_invs invs2 = { }; static struct arm_smmu_invs invs3 = { + .max_invs = 3, .num_invs = 3, .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ @@ -675,6 +678,7 @@ static struct arm_smmu_invs invs3 = { }; static struct arm_smmu_invs invs4 = { + .max_invs = 3, .num_invs = 3, .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, @@ -682,6 +686,7 @@ static struct arm_smmu_invs invs4 = { }; static struct arm_smmu_invs invs5 = { + .max_invs = 3, .num_invs = 3, .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ From e9da5ec4a1d61f6462e43dc9c2bf89247313eebc Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Wed, 2 Sep 2026 09:38:58 +0800 Subject: [PATCH 45/69] dt-bindings: arm-smmu: Document Adreno SMMU for Nord SoC Add specific compatible string to document the GPU SMMU present on Nord SoC, and constrain it to the single HLOS vote clock. Signed-off-by: Shawn Guo Reviewed-by: Bartosz Golaszewski Reviewed-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 187710fcc567..6748351c7812 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -103,6 +103,7 @@ properties: - qcom,kaanapali-smmu-500 - qcom,maili-smmu-500 - qcom,milos-smmu-500 + - qcom,nord-smmu-500 - qcom,qcm2290-smmu-500 - qcom,qcs615-smmu-500 - qcom,qcs8300-smmu-500 @@ -581,6 +582,7 @@ allOf: - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 - qcom,maili-smmu-500 + - qcom,nord-smmu-500 - qcom,sm8750-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 From 84c6089a365ff5547f9ea30cb407dc5fe3124fd3 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:20 -0300 Subject: [PATCH 46/69] iommu/arm-smmu-v3: Handle ARM erratum for CONT under invalidation with SVA The erratum (MMU-700: #3777127, S3: #3673557) deals with under invalidation of a CONT PTE grouping in the SMMU. The recommended work around is to use a range invalidation that spans the entire CONT. The only user of CONT in the kernel right now is through SVA sharing a CPU page table that contains a CONT created by the mm. Previously it was thought that this errata was dealt with because the driver always uses range invalidation. However, there is a subtle detail in the errata that the invalidation range must fully enclose the entire CONT for it to work. It seems that two sequential range invalidations, with a split point falling inside a CONT grouping, will not prevent the errata. The SMMU's range invalidation generation algorithm does not produce a single invalidation for a single SVA invalidation request, nor does the mm carefully align the SVA invalidation ranges to accommodate the splitting of the invalidation into several ranges. Thus, when processing a SVA invalidation, the range invalidation splitting routine can generate a range invalidation that is split in the middle of the CONT and risk under invalidation from this errata. This condition could be triggered by a malicious userspace manipulating the TLB gathers via mmap/mprotect/munmap. Update the errata list to the include the S3 variation, detect the IOMMUs that have it, and then have SVA invalidations use a simplified version of the over invalidation algorithm from the tlbi rework series. This ensures that a single range invalidation is issued for a single MMU notifier callback and now the range invalidation is guarenteed to cover any posible CONT. Future work to add CONT to iommu_domain page tables should either use this one-invalidate/one-range invalidation algorithm or disable CONT support in the iommu_domain. Cc: stable@vger.kernel.org Cc: Vijayanand Jitta Fixes: 3f1ce8e85ee0 ("iommu/arm-smmu-v3: Share process page tables") Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 3 +- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 7 ++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 91 +++++++++++++++---- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 + 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index ac3248b9f2f3..68018bf75b79 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -271,7 +271,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU L1 | #3878312 | N/A | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU S3 | #3995052 | N/A | +| ARM | MMU S3 | #3995052, | N/A | +| | | #3673557 | | +----------------+-----------------+-----------------+-----------------------------+ | ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 0a429c64fbf3..3f50298a1c14 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -215,6 +215,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) if (system_supports_haft()) feat_mask |= ARM_SMMU_FEAT_HAFT; + /* + * The workaround for ARM_SMMU_OPT_FULL_CONT_RANGE_INV requires range + * invalidation support. + */ + if (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV) + feat_mask |= ARM_SMMU_FEAT_RANGE_INV; + if ((smmu->features & feat_mask) != feat_mask) return false; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 658d2cdfa286..502745984a53 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2560,6 +2560,37 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, } } +/* + * Generate a range invalidation for ARM_SMMU_OPT_FULL_CONT_RANGE_INV by + * ensuring the entire SVA requested range is covered with a single range + * invalidation command. The scale is adjusted so that the range invalidation + * may extend past the end of the requested range. This ensures that any CONT + * the MM is invalidating is covered by a single range invalidation. TTL and + * LEAF are always 0 because this is only used by SVA. + */ +static bool arm_smmu_cmdq_batch_add_range_inv(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd, + unsigned long iova, size_t size, + u8 tgsz_lg2) +{ + u64 cur_tg = iova >> tgsz_lg2; + u64 num_tg = ((iova + size - 1) >> tgsz_lg2) - cur_tg + 1; + unsigned int scale = fls64((num_tg - 1) / 32); + + if (scale > 31) + return false; + + cmd->data[0] |= + FIELD_PREP(CMDQ_TLBI_0_NUM, + DIV_ROUND_UP_ULL(num_tg, 1ULL << scale) - 1) | + FIELD_PREP(CMDQ_TLBI_0_SCALE, scale); + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_TG, (tgsz_lg2 - 10) / 2) | + (cur_tg << tgsz_lg2); + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); + return true; +} + static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, size_t granule) { @@ -2587,21 +2618,30 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, - bool leaf, + bool single_range_inv, bool leaf, unsigned long iova, size_t size, unsigned int granule) { - if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) { - struct arm_smmu_cmd nsize_cmd = *cmd; + struct arm_smmu_cmd nsize_cmd; - u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, - CMDQ_0_OP); - arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd); + if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) + goto full_inv; + + if (single_range_inv && size > granule) { + if (!arm_smmu_cmdq_batch_add_range_inv(inv->smmu, cmds, cmd, + iova, size, inv->pgsize)) + goto full_inv; return; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, - iova, size, granule, inv->pgsize); + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, iova, size, + granule, inv->pgsize); + return; + +full_inv: + nsize_cmd = *cmd; + u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, CMDQ_0_OP); + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd); } static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, @@ -2622,7 +2662,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, unsigned long iova, size_t size, - unsigned int granule, bool leaf) + unsigned int granule, + bool single_range_inv, bool leaf) { struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_inv *cur; @@ -2652,14 +2693,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, case INV_TYPE_S1_ASID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, cur->id, 0); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, - iova, size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, + single_range_inv, leaf, iova, + size, granule); break; case INV_TYPE_S2_VMID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, cur->id); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, - iova, size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, + single_range_inv, leaf, iova, + size, granule); break; case INV_TYPE_S2_VMID_S1_CLEAR: /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ @@ -2706,6 +2749,9 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned int granule, bool leaf) { struct arm_smmu_invs *invs; + bool single_range_inv = + smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && + (smmu_domain->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV); /* * An invalidation request must follow some IOPTE change and then load @@ -2745,10 +2791,12 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + __arm_smmu_domain_inv_range(invs, iova, size, granule, + single_range_inv, leaf); read_unlock_irqrestore(&invs->rwlock, flags); } else { - __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + __arm_smmu_domain_inv_range(invs, iova, size, granule, + single_range_inv, leaf); } rcu_read_unlock(); @@ -5081,12 +5129,21 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) /* Arm errata 2268618, 2812531 */ smmu->features &= ~ARM_SMMU_FEAT_NESTING; } + /* Arm errata 3777127 */ + smmu->options |= ARM_SMMU_OPT_FULL_CONT_RANGE_INV; break; case IIDR_PRODUCTID_ARM_MMU_L1: - case IIDR_PRODUCTID_ARM_MMU_S3: - /* Arm errata 3878312/3995052 */ + /* Arm errata 3878312 */ smmu->features &= ~ARM_SMMU_FEAT_BTM; break; + case IIDR_PRODUCTID_ARM_MMU_S3: + /* Arm errata 3995052 */ + smmu->features &= ~ARM_SMMU_FEAT_BTM; + /* Arm errata 3673557 */ + if (variant < 1 || (variant == 1 && revision < 1)) + smmu->options |= + ARM_SMMU_OPT_FULL_CONT_RANGE_INV; + break; } break; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index cbfb3e9f5095..0a1bb75467db 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -940,6 +940,11 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) #define ARM_SMMU_OPT_OVR_INSTCFG_DATA (1 << 5) +/* + * Range invalidation is mandatory and one range invalidation must fully span an + * invalidated CONT + */ +#define ARM_SMMU_OPT_FULL_CONT_RANGE_INV (1 << 6) u32 options; struct arm_smmu_cmdq cmdq; From 0a6cd4ac9252dd4fa27cd42baf00ca6720bc49a8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:21 -0300 Subject: [PATCH 47/69] iommu/arm-smmu-v3: Pass the parameters for the invalidation in a struct These parameters go to a lot of different functions and the next patches will add more. Put them into a struct to keep things tidy. Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 90 ++++++++++----------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 9 +++ 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 502745984a53..bcdeb9a47df7 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2385,8 +2385,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) return IRQ_WAKE_THREAD; } -static struct arm_smmu_cmd -arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) +static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, + struct arm_smmu_tlbi *tlbi) { size_t log2_span; size_t span_mask; @@ -2408,8 +2408,8 @@ arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) * This has the unpleasant side-effect of invalidating all PASID-tagged * ATC entries within the address range. */ - page_start = iova >> inval_grain_shift; - page_end = (iova + size - 1) >> inval_grain_shift; + page_start = tlbi->iova >> inval_grain_shift; + page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift; /* * In an ATS Invalidate Request, the address must be aligned on the @@ -2484,20 +2484,23 @@ static void arm_smmu_tlb_inv_context(void *cookie) static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, bool leaf, - unsigned long iova, size_t size, - size_t granule, size_t pgsize) + struct arm_smmu_cmd *cmd, + struct arm_smmu_tlbi *tlbi, + size_t pgsize) { - unsigned long end = iova + size, num_pages = 0, tg = pgsize; + size_t inv_range = tlbi->iopte_size; + unsigned long iova = tlbi->iova; + unsigned long end = iova + tlbi->size; + unsigned long num_pages = 0; + unsigned int tg = pgsize; u64 orig_data0 = cmd->data[0]; - size_t inv_range = granule; u8 ttl = 0, tg_enc = 0; - if (WARN_ON_ONCE(!size)) + if (WARN_ON_ONCE(!tlbi->size)) return; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - num_pages = size >> tg; + num_pages = tlbi->size >> tg; /* Convert page size of 12,14,16 (log2) to 1,2,3 */ tg_enc = (tg - 10) / 2; @@ -2510,8 +2513,8 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, * want to use a range command, so avoid the SVA corner case * where both scale and num could be 0 as well. */ - if (leaf) - ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); + if (tlbi->leaf_only) + ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tg - 3)); else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1) num_pages++; } @@ -2550,7 +2553,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, * command and something would be very broken if iova had them * set. */ - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) | + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | (iova & ~GENMASK_U64(11, 0)); @@ -2591,13 +2594,13 @@ static bool arm_smmu_cmdq_batch_add_range_inv(struct arm_smmu_device *smmu, return true; } -static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, - size_t granule) +static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, + struct arm_smmu_tlbi *tlbi) { size_t max_tlbi_ops; /* 0 size means invalidate all */ - if (!size || size == SIZE_MAX) + if (!tlbi->size || tlbi->size == SIZE_MAX) return true; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) @@ -2610,32 +2613,31 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, * invalidation feature, where there can be too many per-granule TLBIs, * resulting in a soft lockup. */ - max_tlbi_ops = 1 << (ilog2(granule) - 3); - return size >= max_tlbi_ops * granule; + max_tlbi_ops = 1 << (ilog2(tlbi->iopte_size) - 3); + return tlbi->size >= max_tlbi_ops * tlbi->iopte_size; } /* Used by non INV_TYPE_ATS* invalidations */ static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, - bool single_range_inv, bool leaf, - unsigned long iova, size_t size, - unsigned int granule) + struct arm_smmu_tlbi *tlbi) { struct arm_smmu_cmd nsize_cmd; - if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) + if (arm_smmu_inv_size_too_big(inv->smmu, tlbi)) goto full_inv; - if (single_range_inv && size > granule) { + if (tlbi->has_cont && tlbi->size > tlbi->iopte_size && + (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)) { if (!arm_smmu_cmdq_batch_add_range_inv(inv->smmu, cmds, cmd, - iova, size, inv->pgsize)) + tlbi->iova, tlbi->size, + inv->pgsize)) goto full_inv; return; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, iova, size, - granule, inv->pgsize); + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi, inv->pgsize); return; full_inv: @@ -2660,10 +2662,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, return false; } -static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, - unsigned long iova, size_t size, - unsigned int granule, - bool single_range_inv, bool leaf) +static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_invs *invs) { struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_inv *cur; @@ -2693,20 +2693,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, case INV_TYPE_S1_ASID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, cur->id, 0); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, - single_range_inv, leaf, iova, - size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, cur->id); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, - single_range_inv, leaf, iova, - size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID_S1_CLEAR: /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ - if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) + if (arm_smmu_inv_size_too_big(cur->smmu, tlbi)) break; arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, @@ -2717,7 +2713,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid, - iova, size)); + tlbi)); break; case INV_TYPE_ATS_FULL: arm_smmu_cmdq_batch_add_cmd( @@ -2748,10 +2744,14 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long iova, size_t size, unsigned int granule, bool leaf) { + struct arm_smmu_tlbi tlbi = { + .iova = iova, + .size = size, + .iopte_size = granule, + .has_cont = smmu_domain->stage == ARM_SMMU_DOMAIN_SVA, + .leaf_only = leaf, + }; struct arm_smmu_invs *invs; - bool single_range_inv = - smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && - (smmu_domain->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV); /* * An invalidation request must follow some IOPTE change and then load @@ -2791,12 +2791,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - __arm_smmu_domain_inv_range(invs, iova, size, granule, - single_range_inv, leaf); + __arm_smmu_domain_inv_range(&tlbi, invs); read_unlock_irqrestore(&invs->rwlock, flags); } else { - __arm_smmu_domain_inv_range(invs, iova, size, granule, - single_range_inv, leaf); + __arm_smmu_domain_inv_range(&tlbi, invs); } rcu_read_unlock(); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 0a1bb75467db..f0839cc767dd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -810,6 +810,15 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) return new_invs; } +struct arm_smmu_tlbi { + unsigned long iova; + size_t size; + /* page or block size of the leaf iopte */ + unsigned int iopte_size; + bool has_cont; + bool leaf_only; +}; + struct arm_smmu_evtq { struct arm_smmu_queue q; struct iopf_queue *iopf; From a9573c2847e9a1c3c54df4ed0e14569d1d5ecbfc Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:22 -0300 Subject: [PATCH 48/69] iommu/arm-smmu-v3: Move pgsize out of arm_smmu_inv pgsize is a constant property of the domain, it is the base translation granule of the page table (4k, 16k, 64k) in log2. Store it to the struct arm_smmu_domain based on how the page table was created. Pass it around in the tlbi. Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 1 + drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 29 ++++++++----------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 10 ++++--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 3f50298a1c14..2f76f649dcf1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -345,6 +345,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * ARM_SMMU_FEAT_RANGE_INV is present */ smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->tgsz_lg2 = PAGE_SHIFT; smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index bcdeb9a47df7..2df064cb0a5e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2485,14 +2485,13 @@ static void arm_smmu_tlb_inv_context(void *cookie) static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, - struct arm_smmu_tlbi *tlbi, - size_t pgsize) + struct arm_smmu_tlbi *tlbi) { size_t inv_range = tlbi->iopte_size; unsigned long iova = tlbi->iova; unsigned long end = iova + tlbi->size; unsigned long num_pages = 0; - unsigned int tg = pgsize; + u8 tg = tlbi->tgsz_lg2; u64 orig_data0 = cmd->data[0]; u8 ttl = 0, tg_enc = 0; @@ -2632,12 +2631,12 @@ static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)) { if (!arm_smmu_cmdq_batch_add_range_inv(inv->smmu, cmds, cmd, tlbi->iova, tlbi->size, - inv->pgsize)) + tlbi->tgsz_lg2)) goto full_inv; return; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi, inv->pgsize); + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi); return; full_inv: @@ -2745,6 +2744,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned int granule, bool leaf) { struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, .iova = iova, .size = size, .iopte_size = granule, @@ -3001,6 +3001,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, return -ENOMEM; smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + smmu_domain->tgsz_lg2 = __ffs(pgtbl_cfg.pgsize_bitmap); smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; smmu_domain->domain.geometry.force_aperture = true; if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) @@ -3240,15 +3241,13 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master, static struct arm_smmu_inv * arm_smmu_master_build_inv(struct arm_smmu_master *master, - enum arm_smmu_inv_type type, u32 id, ioasid_t ssid, - size_t pgsize) + enum arm_smmu_inv_type type, u32 id, ioasid_t ssid) { struct arm_smmu_invs *build_invs = master->build_invs; struct arm_smmu_inv *cur, inv = { .smmu = master->smmu, .type = type, .id = id, - .pgsize = pgsize, }; if (WARN_ON(build_invs->num_invs >= build_invs->max_invs)) @@ -3300,28 +3299,24 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, ioasid_t ssid, struct arm_smmu_domain *smmu_domain) { const bool nesting = smmu_domain->nest_parent; - size_t pgsize = 0, i; + size_t i; iommu_group_mutex_assert(master->dev); master->build_invs->num_invs = 0; - /* Range-based invalidation requires the leaf pgsize for calculation */ - if (master->smmu->features & ARM_SMMU_FEAT_RANGE_INV) - pgsize = __ffs(smmu_domain->domain.pgsize_bitmap); - switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_SVA: case ARM_SMMU_DOMAIN_S1: if (!arm_smmu_master_build_inv(master, INV_TYPE_S1_ASID, smmu_domain->cd.asid, - IOMMU_NO_PASID, pgsize)) + IOMMU_NO_PASID)) return NULL; break; case ARM_SMMU_DOMAIN_S2: if (!arm_smmu_master_build_inv(master, INV_TYPE_S2_VMID, smmu_domain->s2_cfg.vmid, - IOMMU_NO_PASID, pgsize)) + IOMMU_NO_PASID)) return NULL; break; default: @@ -3333,7 +3328,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, if (nesting) { if (!arm_smmu_master_build_inv( master, INV_TYPE_S2_VMID_S1_CLEAR, - smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID, 0)) + smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID)) return NULL; } @@ -3344,7 +3339,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, */ if (!arm_smmu_master_build_inv( master, nesting ? INV_TYPE_ATS_FULL : INV_TYPE_ATS, - master->streams[i].id, ssid, 0)) + master->streams[i].id, ssid)) return NULL; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index f0839cc767dd..ae1fc9bae340 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -741,10 +741,9 @@ struct arm_smmu_inv { u8 size_opcode; u8 nsize_opcode; u32 id; /* ASID or VMID or SID */ - union { - size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */ - u32 ssid; /* INV_TYPE_ATS */ - }; + + /* Only used by INV_TYPE_ATS */ + u32 ssid; int users; /* users=0 to mark as a trash to be purged */ }; @@ -815,6 +814,8 @@ struct arm_smmu_tlbi { size_t size; /* page or block size of the leaf iopte */ unsigned int iopte_size; + /* Base Translation Granule of the page table */ + u8 tgsz_lg2; bool has_cont; bool leaf_only; }; @@ -1069,6 +1070,7 @@ struct arm_smmu_domain { spinlock_t devices_lock; bool enforce_cache_coherency : 1; bool nest_parent : 1; + u8 tgsz_lg2; struct mmu_notifier mmu_notifier; }; From 5a158b29766cc91c91e1d9df6aafda19e1924f7b Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:23 -0300 Subject: [PATCH 49/69] iommu/arm-smmu-v3: Optimize range invalidation for latency The server IOMMU drivers focus on invalidation latency by default, over-invalidating if necessary, to round the invalidation range up to a single command. I think this represents a trade off for DMA non-FQ and SVA where stalling the operation is overall worse than re-loading the IOTLB. For instance AMD and VT-d both round the range up to the largest aligned power of two and invalidate that. This causes over-invalidation but that is preferred on real HW over trying to issue a number of smaller range invalidations. SMMUv3 on the other hand will try to do up to 512 single invalidations, otherwise falls back to full invalidation, or it will try to issue an string of range invalidations for every 5 bits of IOVA range to perfectly cover it. While the range-invalidation optimization is pretty good we can still do better and reduce it to only 2 invalidations at maximum using the idea Robin came up with to overlap two range invalidations in the middle. This preserves the exact coverage of the current range invalidation and still caps the number of range invalidations at 2. This works because the range invalidation can start at any IOVA, so we can place a range invalidation forwards from the start and backwards from the end, overlapping in the middle if the scale isn't precise enough. In the cases where this produces 2 range invalidations it continues to produce range invalidations that don't overlap, but the exact split point is different than the original algorithm due to the new calculation method. In cases where the original algorithm would produce 3 or more range invalidations this will produce 2 range invalidations with an overlap. The SVA under invalidation errata work around is maintained by "rounding up" to generate a single range invalidation that over-covers the entire range. Since the normal path is now the only one with a loop, split them into two functions and fold a simplified version of arm_smmu_inv_size_too_big() directly into the normal flow in a way that directly limits the number of single invalidation commands generated, again focusing on controlling latency. The end result is any gather is converted into either: - One invalidate all - One or two range invalidation operations - At most 512 single invalidation ops The current code switches at 511, but this turns every table removal into a full invalidation. Change to 512 to try to minimize the amount of full invalidation when using a 4k translation granule. arm_smmu_domain_inv() is modified to always pass in the tgsz because the new logic relies on it being valid. Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 372 ++++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 16 +- 2 files changed, 238 insertions(+), 150 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 2df064cb0a5e..777ec0e8cb07 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2482,167 +2482,232 @@ static void arm_smmu_tlb_inv_context(void *cookie) arm_smmu_domain_inv(smmu_domain); } -static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, +/* + * Check address alignment for TTL hint per SMMUv3 H.a Section 4.4.1. Address + * bits below the alignment must be zero, otherwise UNPREDICTABLE. + */ +static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg, + unsigned int ttl) +{ + unsigned int pgsz_lg2 = arm_smmu_pt_level_to_lg2sz(tg, 3 - ttl); + + return !(address & GENMASK_U64(pgsz_lg2 - 1, 0)); +} + +struct arm_smmu_range_inv { + u64 start_tg; + /* Normal integer, not encoded. 0 means 0.*/ + u64 num; + unsigned int scale; +}; + +static unsigned int arm_smmu_range_inv_calc_scale(u64 num_tg) +{ + return fls64((num_tg - 1) / (CMDQ_TLBI_RANGE_NUM_MAX + 1)); +} + +static u64 arm_smmu_range_inv_calc_num(u64 num_tg, unsigned int scale) +{ + return DIV_ROUND_UP_ULL(num_tg, 1ULL << scale); +} + +/* + * Initialize the smallest range invalidation covering num_tg and ending at + * last_tg. + */ +static struct arm_smmu_range_inv arm_smmu_range_inv_init_end(u64 last_tg, + u64 num_tg) +{ + struct arm_smmu_range_inv range_inv = {}; + + if (!num_tg) + return range_inv; + + range_inv.scale = arm_smmu_range_inv_calc_scale(num_tg); + range_inv.num = arm_smmu_range_inv_calc_num(num_tg, range_inv.scale); + range_inv.start_tg = last_tg - ((range_inv.num << range_inv.scale) - 1); + return range_inv; +} + +static void arm_smmu_cmdq_batch_add_range_inv( + struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *ref_cmd, bool leaf_only, + const struct arm_smmu_range_inv *range_inv, u8 ttl, u8 tg_enc) +{ + struct arm_smmu_cmd cmd; + unsigned int tgsz_lg2 = tg_enc * 2 + 10; + u64 iova = range_inv->start_tg << tgsz_lg2; + unsigned int num = range_inv->num - 1; + + /* 16K granule TTL=1 is reserved (Section 4.4.1) */ + if (WARN_ON(tgsz_lg2 == 14 && ttl == 1)) + ttl = 0; + + /* Verify address alignment for the TTL hint */ + if (ttl && !arm_smmu_ttl_addr_aligned(iova, tgsz_lg2, ttl)) + ttl = 0; + + /* + * SMMUv3 H.a Section 4.4.1: TG!=0, NUM==0, SCALE==0, TTL==0 is Reserved + * and causes CERROR_ILL. Single tg uses NUM=0, SCALE=0 with a TTL hint + * to target only the exact leaf entry. + * + * For a single tg invalidation a 0 TTL can come from places like the + * SVA path that don't have enough information to get a TTL, or as a + * side of effect of the splitting. + * + * A single-TG invalidation cannot reach this point if it is part of a + * CONT group, so it is safe to transform it into a single invalidation. + * The ARM_SMMU_OPT_FULL_CONT_RANGE_INV errata does not apply. + */ + if (!num && !range_inv->scale && !ttl) + tg_enc = 0; + + cmd.data[0] = ref_cmd->data[0] | FIELD_PREP(CMDQ_TLBI_0_NUM, num) | + FIELD_PREP(CMDQ_TLBI_0_SCALE, range_inv->scale); + cmd.data[1] = ref_cmd->data[1] | + FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf_only) | + FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | + FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | iova; + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &cmd); +} + +/* + * Issue up to two range TLBI commands covering [iova, iova+size). Returns true + * if successful, false if the range is too large to fit into range invalidation + * commands. + * + * Normally the first range invalidation is the largest representable span which + * does not exceed the requested range. If necessary, the second range + * invalidation is the smallest representable range covering the remainder and + * is anchored at the end. Any excess coverage from the second range + * invalidation overlaps the first instead of exceeding the requested range. + * + * If a SVA is being invalidated and the SMMU has the + * ARM_SMMU_OPT_FULL_CONT_RANGE_INV errata this produces only a single range + * invalidation and overinvalidates to ensure any potential CONT is covered with + * a single range invalidation. + */ +static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, struct arm_smmu_tlbi *tlbi) { - size_t inv_range = tlbi->iopte_size; - unsigned long iova = tlbi->iova; - unsigned long end = iova + tlbi->size; - unsigned long num_pages = 0; - u8 tg = tlbi->tgsz_lg2; - u64 orig_data0 = cmd->data[0]; - u8 ttl = 0, tg_enc = 0; + u8 tgsz_lg2 = tlbi->tgsz_lg2; + struct arm_smmu_range_inv first = { .start_tg = tlbi->iova >> + tgsz_lg2 }; + u64 last_tg = (tlbi->iova + tlbi->size - 1) >> tgsz_lg2; + u64 num_tg = last_tg - first.start_tg + 1; + u8 tg_enc = (tgsz_lg2 - 10) / 2; + struct arm_smmu_range_inv trail; + u8 ttl = 0; - if (WARN_ON_ONCE(!tlbi->size)) - return; - - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - num_pages = tlbi->size >> tg; - - /* Convert page size of 12,14,16 (log2) to 1,2,3 */ - tg_enc = (tg - 10) / 2; - - /* - * Determine what level the granule is at. For non-leaf, both - * io-pgtable and SVA pass a nominal last-level granule because - * they don't know what level(s) actually apply, so ignore that - * and leave TTL=0. However for various errata reasons we still - * want to use a range command, so avoid the SVA corner case - * where both scale and num could be 0 as well. - */ - if (tlbi->leaf_only) - ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tg - 3)); - else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1) - num_pages++; - } - - while (iova < end) { - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - /* - * On each iteration of the loop, the range is 5 bits - * worth of the aligned size remaining. - * The range in pages is: - * - * range = (num_pages & (0x1f << __ffs(num_pages))) - */ - unsigned long scale, num; - - /* Determine the power of 2 multiple number of pages */ - scale = __ffs(num_pages); - - /* Determine how many chunks of 2^scale size we have */ - num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; - - /* Keep the pre-DS 5-bit truncation when scale > 31 */ - cmd->data[0] = orig_data0 | - FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) | - FIELD_PREP(CMDQ_TLBI_0_SCALE, scale & 0x1f); - - /* range is num * 2^scale * pgsize */ - inv_range = num << (scale + tg); - - /* Clear out the lower order bits for the next iteration */ - num_pages -= num << scale; - } - - /* - * IPA has fewer bits than VA, but they are reserved in the - * command and something would be very broken if iova had them - * set. - */ - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | - FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | - FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | - (iova & ~GENMASK_U64(11, 0)); - - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); - iova += inv_range; - } -} - -/* - * Generate a range invalidation for ARM_SMMU_OPT_FULL_CONT_RANGE_INV by - * ensuring the entire SVA requested range is covered with a single range - * invalidation command. The scale is adjusted so that the range invalidation - * may extend past the end of the requested range. This ensures that any CONT - * the MM is invalidating is covered by a single range invalidation. TTL and - * LEAF are always 0 because this is only used by SVA. - */ -static bool arm_smmu_cmdq_batch_add_range_inv(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, - unsigned long iova, size_t size, - u8 tgsz_lg2) -{ - u64 cur_tg = iova >> tgsz_lg2; - u64 num_tg = ((iova + size - 1) >> tgsz_lg2) - cur_tg + 1; - unsigned int scale = fls64((num_tg - 1) / 32); - - if (scale > 31) - return false; - - cmd->data[0] |= - FIELD_PREP(CMDQ_TLBI_0_NUM, - DIV_ROUND_UP_ULL(num_tg, 1ULL << scale) - 1) | - FIELD_PREP(CMDQ_TLBI_0_SCALE, scale); - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_TG, (tgsz_lg2 - 10) / 2) | - (cur_tg << tgsz_lg2); - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); - return true; -} - -static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, - struct arm_smmu_tlbi *tlbi) -{ - size_t max_tlbi_ops; - - /* 0 size means invalidate all */ - if (!tlbi->size || tlbi->size == SIZE_MAX) - return true; - - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) + if (!tlbi->size) return false; /* - * Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, - * this is used as a threshold to replace "size_opcode" commands with a - * single "nsize_opcode" command, when SMMU doesn't implement the range - * invalidation feature, where there can be too many per-granule TLBIs, - * resulting in a soft lockup. + * Determine what level the granule is at. For non-leaf, both io-pgtable + * and SVA pass a nominal last-level granule because they don't know + * what level(s) actually apply, so leave TTL=0. */ - max_tlbi_ops = 1 << (ilog2(tlbi->iopte_size) - 3); - return tlbi->size >= max_tlbi_ops * tlbi->iopte_size; + if (tlbi->leaf_only) + ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tgsz_lg2 - 3)); + + /* + * The spec defines the invalidated range as: + * Range = ((NUM+1) * 2^SCALE) * Translation_Granule_Size + * NUM is 5 bits, so (NUM+1) covers 1..32 granules. Find the smallest + * SCALE at which a single command could cover num_tg. + * + * Unlike other IOMMUs the spec has no alignment requirement on the + * address beyond alignment to tg (so long as TTL=0). + */ + first.scale = arm_smmu_range_inv_calc_scale(num_tg); + if (first.scale > 31) { + /* Range too large for a single command do full invalidation */ + return false; + } + + if (tlbi->has_cont && + (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)) { + /* + * Produce a single invalidation by rounding up and disabling + * the trailer. + */ + first.num = arm_smmu_range_inv_calc_num(num_tg, first.scale); + trail.num = 0; + } else { + /* + * Produce two invalidations by rounding down and adding a + * second trailing range invalidation anchored at the end. + */ + first.num = num_tg >> first.scale; + trail = arm_smmu_range_inv_init_end( + last_tg, num_tg - ((u64)first.num << first.scale)); + } + arm_smmu_cmdq_batch_add_range_inv(smmu, cmds, cmd, tlbi->leaf_only, + &first, ttl, tg_enc); + + if (trail.num) + arm_smmu_cmdq_batch_add_range_inv( + smmu, cmds, cmd, tlbi->leaf_only, &trail, ttl, tg_enc); + return true; } -/* Used by non INV_TYPE_ATS* invalidations */ -static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, +/* + * One TLBI command per IOTLB entry, assuming the entries are all at least + * iopte_granule sized. Returns false if too many commands would be needed which + * indicates too high a latency. The threshold is similar to MAX_DVM_OPS in + * arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE. + */ +static bool arm_smmu_cmdq_batch_add_single(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd, + struct arm_smmu_tlbi *tlbi) +{ + unsigned long num_ops = tlbi->size / tlbi->iopte_size; + unsigned long iova = tlbi->iova; + unsigned long i; + + if (!num_ops || num_ops > 512) + return false; + + for (i = 0; i < num_ops; i++) { + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | + (iova & ~GENMASK_U64(11, 0)); + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); + iova += tlbi->iopte_size; + } + return true; +} + +static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd) +{ + u64p_replace_bits(&cmd->data[0], inv->nsize_opcode, CMDQ_0_OP); + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd); +} + +/* + * Used by non INV_TYPE_ATS* invalidations. Returns true if it fell back to + * full invalidation using nsize_opcode. + */ +static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, struct arm_smmu_tlbi *tlbi) { - struct arm_smmu_cmd nsize_cmd; - - if (arm_smmu_inv_size_too_big(inv->smmu, tlbi)) - goto full_inv; - - if (tlbi->has_cont && tlbi->size > tlbi->iopte_size && - (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)) { - if (!arm_smmu_cmdq_batch_add_range_inv(inv->smmu, cmds, cmd, - tlbi->iova, tlbi->size, - tlbi->tgsz_lg2)) - goto full_inv; - return; + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + if (arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi)) + return false; + } else { + if (arm_smmu_cmdq_batch_add_single(inv->smmu, cmds, cmd, tlbi)) + return false; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi); - return; - -full_inv: - nsize_cmd = *cmd; - u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, CMDQ_0_OP); - arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd); + arm_smmu_inv_all_cmd(inv, cmds, cmd); + return true; } static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, @@ -2664,6 +2729,7 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi, struct arm_smmu_invs *invs) { + struct arm_smmu_inv *used_s12_vmall = NULL; struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_inv *cur; struct arm_smmu_inv *end; @@ -2695,13 +2761,21 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi, arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID: - cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, - 0, cur->id); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); + cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, + cur->id); + if (arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi)) + used_s12_vmall = cur + 1; break; case INV_TYPE_S2_VMID_S1_CLEAR: - /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ - if (arm_smmu_inv_size_too_big(cur->smmu, tlbi)) + /* + * S2_VMID used CMDQ_OP_TLBI_S12_VMALL which already + * flushed S1 entries. These two types always come in + * pairs and arm_smmu_inv_cmp() ensures that they are + * consecutive in the list for the same SMMU. There may + * be several pairings so check this is paired with the + * one that did the full invalidation. + */ + if (used_s12_vmall == cur) break; arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ae1fc9bae340..d2dc787349e0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -809,6 +809,19 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) return new_invs; } +/* Generic page-table level 0 is the leaf-only level. */ +static inline unsigned int arm_smmu_pt_level_to_lg2sz(unsigned int tgsz_lg2, + unsigned int level) +{ + return tgsz_lg2 + (tgsz_lg2 - ilog2(sizeof(u64))) * level; +} + +static inline unsigned int arm_smmu_pt_lg2sz_to_level(unsigned int tgsz_lg2, + unsigned int lg2sz) +{ + return (lg2sz - tgsz_lg2) / (tgsz_lg2 - ilog2(sizeof(u64))); +} + struct arm_smmu_tlbi { unsigned long iova; size_t size; @@ -1182,7 +1195,8 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain) { - arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false); + arm_smmu_domain_inv_range(smmu_domain, 0, 0, 1 << smmu_domain->tgsz_lg2, + false); } void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, From fc62cd069cf7893e834fbedcd8c4c0a6a2109728 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:24 -0300 Subject: [PATCH 50/69] iommu/arm-smmu-v3: Keep track in arm_smmu_invs if range invalidation is used Summarize if any of the inv entries will use range invalidation and if any need ARM_SMMU_OPT_FULL_CONT_RANGE_INV. The next patch will use this to avoid range invalidation pre-calculations unless the range commands will be used. Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 38 +++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 19 ++++++++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +++ 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index e52a7d95c919..5a4e45e15125 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -24,7 +24,9 @@ struct arm_smmu_test_writer { static struct arm_smmu_ste bypass_ste; static struct arm_smmu_ste abort_ste; static struct arm_smmu_device smmu = { - .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR + .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR | + ARM_SMMU_FEAT_RANGE_INV, + .options = ARM_SMMU_OPT_FULL_CONT_RANGE_INV, }; static struct mm_struct sva_mm = { .pgd = (void *)0xdaedbeefdeadbeefULL, @@ -645,6 +647,8 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, { KUNIT_ASSERT_EQ(test, invs->num_invs, num_invs); KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); + KUNIT_EXPECT_TRUE(test, invs->has_range_inv); + KUNIT_EXPECT_TRUE(test, invs->has_full_cont_range_inv); while (num_invs--) { KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users), @@ -656,41 +660,41 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, static struct arm_smmu_invs invs1 = { .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, - { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, - { .type = INV_TYPE_ATS, .id = 3, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 3, }, }, }; static struct arm_smmu_invs invs2 = { .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 4, }, - { .type = INV_TYPE_ATS, .id = 5, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 4, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, }, }; static struct arm_smmu_invs invs3 = { .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ - { .type = INV_TYPE_ATS, .id = 6, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 6, }, }, }; static struct arm_smmu_invs invs4 = { .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, - { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, }; static struct arm_smmu_invs invs5 = { .max_invs = 3, .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ - { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, }; static void arm_smmu_v3_invs_test(struct kunit *test) @@ -711,6 +715,8 @@ static void arm_smmu_v3_invs_test(struct kunit *test) test_a = arm_smmu_invs_alloc(0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a); KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + KUNIT_EXPECT_FALSE(test, test_a->has_range_inv); + KUNIT_EXPECT_FALSE(test, test_a->has_full_cont_range_inv); /* Test1: merge invs1 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs1); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 777ec0e8cb07..7a5f4b54f306 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1059,6 +1059,19 @@ static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]); } +static void arm_smmu_invs_update_caps(struct arm_smmu_invs *invs, + const struct arm_smmu_inv *inv) +{ + if (arm_smmu_inv_is_ats(inv)) + invs->has_ats = true; + + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + invs->has_range_inv = true; + if (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV) + invs->has_full_cont_range_inv = true; + } +} + /** * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for * arm_smmu_invs_merge() or arm_smmu_invs_unref() @@ -1129,8 +1142,7 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, */ if (new != new_invs->inv) WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1); - if (arm_smmu_inv_is_ats(new)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, new); new++; } @@ -1240,8 +1252,7 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) arm_smmu_invs_for_each_entry(invs, i, inv) { new_invs->inv[num_invs] = *inv; - if (arm_smmu_inv_is_ats(inv)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, inv); num_invs++; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index d2dc787349e0..8b71e6cc19f2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -763,6 +763,9 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) * Must not be greater than @num_invs * @rwlock: optional rwlock to fence ATS operations * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL + * @has_range_inv: flag if any entry's SMMU supports range invalidation + * @has_full_cont_range_inv: flag if any entry's SMMU requires the CONT range + * invalidation workaround * @rcu: rcu head for kfree_rcu() * @inv: flexible invalidation array * @@ -792,6 +795,8 @@ struct arm_smmu_invs { size_t num_trashes; rwlock_t rwlock; bool has_ats; + bool has_range_inv; + bool has_full_cont_range_inv; struct rcu_head rcu; struct arm_smmu_inv inv[] __counted_by(max_invs); }; From bc5c3475d63c6df518679902ea7c5396ac2a8692 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:25 -0300 Subject: [PATCH 51/69] iommu/arm-smmu-v3: Precompute the invalidation commands Store the required cmd data in the tlbi and just copy it out when processing each item in the invs list. The cmd form only depends on if the instance supports range invalidation or not, otherwise it is always the same. This avoids redundant calculations for each invs entry. Reviewed-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 147 +++++++++++--------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 12 +- 2 files changed, 94 insertions(+), 65 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 7a5f4b54f306..26490d117672 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2540,12 +2540,12 @@ static struct arm_smmu_range_inv arm_smmu_range_inv_init_end(u64 last_tg, return range_inv; } -static void arm_smmu_cmdq_batch_add_range_inv( - struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *ref_cmd, bool leaf_only, - const struct arm_smmu_range_inv *range_inv, u8 ttl, u8 tg_enc) +static void +arm_smmu_tlbi_add_range_cmd(struct arm_smmu_tlbi *tlbi, + const struct arm_smmu_range_inv *range_inv, u8 ttl, + u8 tg_enc) { - struct arm_smmu_cmd cmd; + struct arm_smmu_cmd *cmd = &tlbi->range.cmds[tlbi->range.num_cmds++]; unsigned int tgsz_lg2 = tg_enc * 2 + 10; u64 iova = range_inv->start_tg << tgsz_lg2; unsigned int num = range_inv->num - 1; @@ -2574,19 +2574,16 @@ static void arm_smmu_cmdq_batch_add_range_inv( if (!num && !range_inv->scale && !ttl) tg_enc = 0; - cmd.data[0] = ref_cmd->data[0] | FIELD_PREP(CMDQ_TLBI_0_NUM, num) | - FIELD_PREP(CMDQ_TLBI_0_SCALE, range_inv->scale); - cmd.data[1] = ref_cmd->data[1] | - FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf_only) | - FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | - FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | iova; - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &cmd); + cmd->data[0] = FIELD_PREP(CMDQ_TLBI_0_NUM, num) | + FIELD_PREP(CMDQ_TLBI_0_SCALE, range_inv->scale); + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | + FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | + FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | iova; } /* - * Issue up to two range TLBI commands covering [iova, iova+size). Returns true - * if successful, false if the range is too large to fit into range invalidation - * commands. + * Generate up to two range TLBI command payloads covering [iova, iova+size). + * Sets use_full_inv if the range is too large to represent. * * Normally the first range invalidation is the largest representable span which * does not exceed the requested range. If necessary, the second range @@ -2594,15 +2591,12 @@ static void arm_smmu_cmdq_batch_add_range_inv( * is anchored at the end. Any excess coverage from the second range * invalidation overlaps the first instead of exceeding the requested range. * - * If a SVA is being invalidated and the SMMU has the - * ARM_SMMU_OPT_FULL_CONT_RANGE_INV errata this produces only a single range - * invalidation and overinvalidates to ensure any potential CONT is covered with - * a single range invalidation. + * For SVA on an invs containing an SMMU with ARM_SMMU_OPT_FULL_CONT_RANGE_INV, + * produce only a single range invalidation and overinvalidate so any potential CONT is + * covered by one command. */ -static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, - struct arm_smmu_tlbi *tlbi) +static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi, + bool single_range_inv) { u8 tgsz_lg2 = tlbi->tgsz_lg2; struct arm_smmu_range_inv first = { .start_tg = tlbi->iova >> @@ -2613,9 +2607,6 @@ static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_range_inv trail; u8 ttl = 0; - if (!tlbi->size) - return false; - /* * Determine what level the granule is at. For non-leaf, both io-pgtable * and SVA pass a nominal last-level granule because they don't know @@ -2636,11 +2627,11 @@ static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, first.scale = arm_smmu_range_inv_calc_scale(num_tg); if (first.scale > 31) { /* Range too large for a single command do full invalidation */ - return false; + tlbi->range.use_full_inv = true; + return; } - if (tlbi->has_cont && - (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)) { + if (single_range_inv) { /* * Produce a single invalidation by rounding up and disabling * the trailer. @@ -2656,40 +2647,27 @@ static bool arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, trail = arm_smmu_range_inv_init_end( last_tg, num_tg - ((u64)first.num << first.scale)); } - arm_smmu_cmdq_batch_add_range_inv(smmu, cmds, cmd, tlbi->leaf_only, - &first, ttl, tg_enc); + arm_smmu_tlbi_add_range_cmd(tlbi, &first, ttl, tg_enc); if (trail.num) - arm_smmu_cmdq_batch_add_range_inv( - smmu, cmds, cmd, tlbi->leaf_only, &trail, ttl, tg_enc); - return true; + arm_smmu_tlbi_add_range_cmd(tlbi, &trail, ttl, tg_enc); } /* * One TLBI command per IOTLB entry, assuming the entries are all at least - * iopte_granule sized. Returns false if too many commands would be needed which - * indicates too high a latency. The threshold is similar to MAX_DVM_OPS in - * arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE. + * iopte_granule sized. Sets use_full_inv if too many commands would be needed + * which indicates too high a latency. The threshold is similar to MAX_DVM_OPS + * in arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE. */ -static bool arm_smmu_cmdq_batch_add_single(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, - struct arm_smmu_tlbi *tlbi) +static void arm_smmu_tlbi_calc_single(struct arm_smmu_tlbi *tlbi) { unsigned long num_ops = tlbi->size / tlbi->iopte_size; - unsigned long iova = tlbi->iova; - unsigned long i; - if (!num_ops || num_ops > 512) - return false; - - for (i = 0; i < num_ops; i++) { - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | - (iova & ~GENMASK_U64(11, 0)); - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); - iova += tlbi->iopte_size; + if (!num_ops || num_ops > 512) { + tlbi->single.use_full_inv = true; + return; } - return true; + tlbi->single.num = num_ops; } static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv, @@ -2709,16 +2687,37 @@ static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmd *cmd, struct arm_smmu_tlbi *tlbi) { + u64 iova = tlbi->iova; + unsigned int i; + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - if (arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi)) - return false; - } else { - if (arm_smmu_cmdq_batch_add_single(inv->smmu, cmds, cmd, tlbi)) - return false; + if (tlbi->range.use_full_inv) { + arm_smmu_inv_all_cmd(inv, cmds, cmd); + return true; + } + for (i = 0; i < tlbi->range.num_cmds; i++) { + struct arm_smmu_cmd range_cmd = tlbi->range.cmds[i]; + + range_cmd.data[0] |= cmd->data[0]; + range_cmd.data[1] |= cmd->data[1]; + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, + &range_cmd); + } + return false; } - arm_smmu_inv_all_cmd(inv, cmds, cmd); - return true; + if (tlbi->single.use_full_inv) { + arm_smmu_inv_all_cmd(inv, cmds, cmd); + return true; + } + + for (i = 0; i < tlbi->single.num; i++) { + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | + (iova & ~GENMASK_U64(11, 0)); + iova += tlbi->iopte_size; + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd); + } + return false; } static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, @@ -2737,8 +2736,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, return false; } -static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi, - struct arm_smmu_invs *invs) +static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_invs *invs) { struct arm_smmu_inv *used_s12_vmall = NULL; struct arm_smmu_cmdq_batch cmds = {}; @@ -2833,11 +2832,17 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, .iova = iova, .size = size, .iopte_size = granule, - .has_cont = smmu_domain->stage == ARM_SMMU_DOMAIN_SVA, .leaf_only = leaf, }; struct arm_smmu_invs *invs; + if (!size || size == SIZE_MAX) { + tlbi.single.use_full_inv = true; + tlbi.range.use_full_inv = true; + } else { + arm_smmu_tlbi_calc_single(&tlbi); + } + /* * An invalidation request must follow some IOPTE change and then load * an invalidation array. In the meantime, a domain attachment mutates @@ -2868,6 +2873,20 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, rcu_read_lock(); invs = rcu_dereference(smmu_domain->invs); + /* + * Only precompute range invalidation commands when they will be used. + * The invs generation ensures this matches the instances being + * invalidated. + */ + if (invs->has_range_inv) { + if (!tlbi.range.use_full_inv) { + arm_smmu_tlbi_calc_range( + &tlbi, + smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && + invs->has_full_cont_range_inv); + } + } + /* * Avoid locking unless ATS is being used. No ATC invalidation can be * going on after a domain is detached. @@ -2876,10 +2895,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - __arm_smmu_domain_inv_range(&tlbi, invs); + arm_smmu_domain_tlbi_inv(&tlbi, invs); read_unlock_irqrestore(&invs->rwlock, flags); } else { - __arm_smmu_domain_inv_range(&tlbi, invs); + arm_smmu_domain_tlbi_inv(&tlbi, invs); } rcu_read_unlock(); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 8b71e6cc19f2..bc520698863e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -834,8 +834,18 @@ struct arm_smmu_tlbi { unsigned int iopte_size; /* Base Translation Granule of the page table */ u8 tgsz_lg2; - bool has_cont; bool leaf_only; + + struct { + bool use_full_inv; + u16 num; + } single; + + struct { + bool use_full_inv; + u8 num_cmds; + struct arm_smmu_cmd cmds[2]; + } range; }; struct arm_smmu_evtq { From d587faf668287f0c18f016da29c3bc05a87efa1d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 2 Oct 2026 22:23:26 -0300 Subject: [PATCH 52/69] iommu/arm-smmu-v3: Populate the tlbi at the top of the call chain Each of these have their own unique situation, populate the tlbi right at the top and pass it into arm_smmu_domain_inv_range(). They will diverge further when the iommupt invalidation scheme is introduced. Reviewed-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Tested-by: Nicolin Chen Tested-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 21 ++++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 60 +++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 15 +++-- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 2f76f649dcf1..faeec656da9e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -140,16 +140,19 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, { struct arm_smmu_domain *smmu_domain = container_of(mn, struct arm_smmu_domain, mmu_notifier); - size_t size; + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = start, + /* + * The mm_types defines vm_end as the first byte after the end + * address, different from IOMMU subsystem using the last + * address of an address range. + */ + .size = end - start, + .iopte_size = PAGE_SIZE, + }; - /* - * The mm_types defines vm_end as the first byte after the end address, - * different from IOMMU subsystem using the last address of an address - * range. So do a simple translation here by calculating size correctly. - */ - size = end - start; - - arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 26490d117672..9724c58d2194 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2823,25 +2823,19 @@ static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi, } } -void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size, - unsigned int granule, bool leaf) +/* + * Perform the invalidation with the parameters described by tlbi on everything + * caching for smmu_domain. tlbi is an on-stack structure that contains both the + * input description and working memory for this function it must be discarded + * after this returns. + */ +void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_domain *smmu_domain) { - struct arm_smmu_tlbi tlbi = { - .tgsz_lg2 = smmu_domain->tgsz_lg2, - .iova = iova, - .size = size, - .iopte_size = granule, - .leaf_only = leaf, - }; struct arm_smmu_invs *invs; - if (!size || size == SIZE_MAX) { - tlbi.single.use_full_inv = true; - tlbi.range.use_full_inv = true; - } else { - arm_smmu_tlbi_calc_single(&tlbi); - } + if (!tlbi->single.use_full_inv) + arm_smmu_tlbi_calc_single(tlbi); /* * An invalidation request must follow some IOPTE change and then load @@ -2860,7 +2854,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, * * [CPU0] | [CPU1] * change IOPTE on new domain: | - * arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs() + * arm_smmu_domain_tlbi() { | arm_smmu_install_new_domain_invs() * smp_mb(); // ensures IOPTE | arm_smmu_install_ste_for_dev { * // seen by SMMU | dma_wmb(); // ensures invs update * // load the updated invs | // before updating STE @@ -2879,9 +2873,9 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, * invalidated. */ if (invs->has_range_inv) { - if (!tlbi.range.use_full_inv) { + if (!tlbi->range.use_full_inv) { arm_smmu_tlbi_calc_range( - &tlbi, + tlbi, smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && invs->has_full_cont_range_inv); } @@ -2895,10 +2889,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - arm_smmu_domain_tlbi_inv(&tlbi, invs); + arm_smmu_domain_tlbi_inv(tlbi, invs); read_unlock_irqrestore(&invs->rwlock, flags); } else { - arm_smmu_domain_tlbi_inv(&tlbi, invs); + arm_smmu_domain_tlbi_inv(tlbi, invs); } rcu_read_unlock(); @@ -2914,12 +2908,23 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, iommu_iotlb_gather_add_page(domain, gather, iova, granule); } +/* + * Called by io-pgtable-arm.c for each single table level it wants to remove. + * size is the size of the table level and granule is the tg in bytes. This must + * clear the walk cache and any leaves within the range. + */ static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, size_t granule, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = iova, + .size = size, + .iopte_size = 1 << smmu_domain->tgsz_lg2, + }; - arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static const struct iommu_flush_ops arm_smmu_flush_ops = { @@ -4192,13 +4197,18 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .iova = gather->start, + .size = gather->end - gather->start + 1, + .iopte_size = gather->pgsize, + .leaf_only = true, + }; if (!gather->pgsize) return; - arm_smmu_domain_inv_range(smmu_domain, gather->start, - gather->end - gather->start + 1, - gather->pgsize, true); + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } static phys_addr_t diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index bc520698863e..8e205ddd6bf6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1204,14 +1204,19 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, struct arm_smmu_domain *smmu_domain, ioasid_t pasid, struct arm_smmu_cd *cd, struct iommu_domain *old); -void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size, - unsigned int granule, bool leaf); +void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_domain *smmu_domain); static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain) { - arm_smmu_domain_inv_range(smmu_domain, 0, 0, 1 << smmu_domain->tgsz_lg2, - false); + /* Prefilled for invalidate all */ + struct arm_smmu_tlbi tlbi = { + .tgsz_lg2 = smmu_domain->tgsz_lg2, + .single.use_full_inv = true, + .range.use_full_inv = true, + }; + + arm_smmu_domain_tlbi(&tlbi, smmu_domain); } void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, From ab0dbf411aced59c04ce86fd8443faf13bdc9936 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 15 Sep 2026 09:38:13 -0700 Subject: [PATCH 53/69] iommu/arm-smmu-v3: Disable the impl before disabling the SMMU on shutdown arm_smmu_device_shutdown() only calls arm_smmu_device_disable(), and never the impl_ops->device_disable() hook. On tegra241-cmdqv, that hook quiesces every VINTF through tegra241_vintf_hw_deinit(), which issues a CMD_SYNC on the shared CMDQ while it is still enabled. arm_smmu_disable_action() already runs this hook before disabling the SMMU, to cover an unbind or a probe failure, and no devres release ever follows a reboot or a kexec to run it again. Have arm_smmu_device_shutdown() call it directly, instead of reimplementing part of what it already does. Fixes: 9ff145a25c5c ("iommu/tegra241-cmdqv: Fix CMD_SYNC use-after-free on teardown") Cc: stable@vger.kernel.org Reviewed-by: Jonathan Cameron Assisted-by: LLM Signed-off-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 9724c58d2194..afb34a192b88 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -5874,7 +5874,7 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - arm_smmu_device_disable(smmu); + arm_smmu_disable_action(smmu); } static const struct of_device_id arm_smmu_of_match[] = { From 580bbfcdcd2e2be5e29e19acbe4be44da9be2837 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 15 Sep 2026 09:38:14 -0700 Subject: [PATCH 54/69] iommu/arm-smmu-v3: Add arm_smmu_attach_release() The IOPF teardown is done in arm_smmu_remove_master_domain() when releasing the master_domain on detach, under the global arm_smmu_asid_lock mutex. A later change will add an IOPF workqueue flush to that teardown, which can block on a user-faulting page-fault handler. Holding the arm_smmu_asid_lock across it would stall every unrelated attachment in the system. Split the teardown out of arm_smmu_remove_master_domain(), to a new helper arm_smmu_attach_release() that runs after arm_smmu_asid_lock is released. No functional change: the old master_domain belongs to no other device, so freeing it outside the lock stays safe, still under iommu_group->mutex. Reviewed-by: Jonathan Cameron Signed-off-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- .../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 1 + drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 44 ++++++++++++++----- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 + 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 25982bdbcbd9..fce026efa44f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -194,6 +194,7 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, arm_smmu_install_ste_for_dev(master, &ste); arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); return 0; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index afb34a192b88..ce7f89fd7b33 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3457,9 +3457,9 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, return master->build_invs; } -static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, - struct iommu_domain *domain, - ioasid_t ssid) +static struct arm_smmu_master_domain * +arm_smmu_remove_master_domain(struct arm_smmu_master *master, + struct iommu_domain *domain, ioasid_t ssid) { struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain); struct arm_smmu_master_domain *master_domain; @@ -3467,7 +3467,7 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, unsigned long flags; if (!smmu_domain) - return; + return NULL; if (domain->type == IOMMU_DOMAIN_NESTED) nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats; @@ -3482,8 +3482,24 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + /* arm_smmu_attach_release() will free it */ + return master_domain; +} + +/* Release the old master_domain detached by arm_smmu_remove_master_domain() */ +void arm_smmu_attach_release(struct arm_smmu_attach_state *state) +{ + struct arm_smmu_master_domain *master_domain = state->old_master_domain; + struct arm_smmu_master *master = state->master; + + iommu_group_mutex_assert(master->dev); + + if (!master_domain) + return; + arm_smmu_disable_iopf(master, master_domain); kfree(master_domain); + state->old_master_domain = NULL; } /* @@ -3781,7 +3797,8 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) arm_smmu_atc_inv_master(master, IOMMU_NO_PASID); } - arm_smmu_remove_master_domain(master, state->old_domain, state->ssid); + state->old_master_domain = arm_smmu_remove_master_domain( + master, state->old_domain, state->ssid); arm_smmu_install_old_domain_invs(state); master->ats_enabled = state->ats_enabled; } @@ -3856,6 +3873,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev, arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); return 0; } @@ -3938,8 +3956,10 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, mutex_lock(&arm_smmu_asid_lock); ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain); - if (ret) - goto out_unlock; + if (ret) { + mutex_unlock(&arm_smmu_asid_lock); + return ret; + } /* * We don't want to obtain to the asid_lock too early, so fix up the @@ -3953,10 +3973,9 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, arm_smmu_update_ste(master, sid_domain, state.ats_enabled); arm_smmu_attach_commit(&state); - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); - return ret; + arm_smmu_attach_release(&state); + return 0; } static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, @@ -3976,9 +3995,11 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, arm_smmu_clear_cd(master, pasid); if (master->ats_enabled) arm_smmu_atc_inv_master(master, pasid); - arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid); + state.old_master_domain = arm_smmu_remove_master_domain( + master, &smmu_domain->domain, pasid); arm_smmu_install_old_domain_invs(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); /* * When the last user of the CD table goes away downgrade the STE back @@ -4041,6 +4062,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, arm_smmu_install_ste_for_dev(master, ste); arm_smmu_attach_commit(&state); mutex_unlock(&arm_smmu_asid_lock); + arm_smmu_attach_release(&state); /* * This has to be done after removing the master from the diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 8e205ddd6bf6..20bf45e756fa 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1261,12 +1261,14 @@ struct arm_smmu_attach_state { struct arm_smmu_vmaster *vmaster; struct arm_smmu_inv_state old_domain_invst; struct arm_smmu_inv_state new_domain_invst; + struct arm_smmu_master_domain *old_master_domain; bool ats_enabled; }; int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, struct iommu_domain *new_domain); void arm_smmu_attach_commit(struct arm_smmu_attach_state *state); +void arm_smmu_attach_release(struct arm_smmu_attach_state *state); void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master, const struct arm_smmu_ste *target); From 02b5c4639bcd3a07f1fad7662da32b8b73371597 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 15 Sep 2026 09:38:15 -0700 Subject: [PATCH 55/69] iommu/arm-smmu-v3: Add Q_POS() macro A queue position, the wrap bit combined with the index, is Q_WRP | Q_IDX. It is a (max_n_shift + 1)-bit value that wraps at twice the queue capacity. queue_inc_cons(), queue_sync_cons_ovf(), and queue_inc_prod_n() currently compute such positions by open-coding the two macros at each call site. Add a Q_POS() macro and switch the open-coded sites to it. A subsequent change will build a Q_DIFF() on it, to count the entries that a queue pointer moved past. No functional change intended. Assisted-by: LLM Signed-off-by: Nicolin Chen Reviewed-by: Jonathan Cameron Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 11 +++++------ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 ++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ce7f89fd7b33..7d2415143407 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -187,8 +187,8 @@ static void queue_sync_cons_out(struct arm_smmu_queue *q) static void queue_inc_cons(struct arm_smmu_ll_queue *q) { - u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; - q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); + u32 cons = Q_POS(q, q->cons) + 1; + q->cons = Q_OVF(q->cons) | Q_POS(q, cons); } static void queue_sync_cons_ovf(struct arm_smmu_queue *q) @@ -198,8 +198,7 @@ static void queue_sync_cons_ovf(struct arm_smmu_queue *q) if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons))) return; - llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | - Q_IDX(llq, llq->cons); + llq->cons = Q_OVF(llq->prod) | Q_POS(llq, llq->cons); queue_sync_cons_out(q); } @@ -224,8 +223,8 @@ static int queue_sync_prod_in(struct arm_smmu_queue *q) static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n) { - u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; - return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); + u32 prod = Q_POS(q, q->prod) + n; + return Q_OVF(q->prod) | Q_POS(q, prod); } static void queue_poll_init(struct arm_smmu_device *smmu, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 20bf45e756fa..bb048e08d613 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -188,6 +188,8 @@ struct arm_vsmmu; #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1)) #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift)) +/* A position is Q_WRP | Q_IDX, wrapping at twice the queue capacity */ +#define Q_POS(llq, p) (Q_WRP(llq, p) | Q_IDX(llq, p)) #define Q_OVERFLOW_FLAG (1U << 31) #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG) #define Q_ENT(q, p) ((q)->base + \ From 2888520bdcbe0599cfc77b2fad8dc6c95505a3cb Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 23 Sep 2026 13:11:25 -0700 Subject: [PATCH 56/69] iommu/arm-smmu-v3: Don't rb_erase() a never-inserted stream node arm_smmu_insert_master() skips inserting a stream whose StreamID duplicates one the same master already owns (bridged PCI devices can present duplicate IDs), leaving that master->streams[i].node zeroed and unlinked from the smmu->streams rb-tree. Both the insert error-rollback loop and arm_smmu_remove_master() then call rb_erase() on every master->streams[i].node unconditionally. rb_erase() on a zeroed node sees a NULL parent, treats the node as the tree root and sets root->rb_node = NULL, silently emptying the whole SID tree and breaking SID lookups (and DMA) for every other master on the SMMU. Mark each node with RB_CLEAR_NODE() after sort_nonatomic() reorders the array, since sorting relocates the entries and would leave the earlier self-referential RB_CLEAR_NODE() pointer stale. An un-inserted node then stays RB_EMPTY_NODE() and is skipped in both erase loops; inserted nodes are linked by rb_find_add() and erased as before. Fixes: b00d24997a11 ("iommu/arm-smmu-v3: Fix iommu_device_probe bug due to duplicated stream ids") Assisted-by: LLM Signed-off-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 7d2415143407..34e916ea339f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4320,6 +4320,13 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, sizeof(master->streams[0]), arm_smmu_stream_id_cmp, NULL); + /* + * Clear after sorting: RB_CLEAR_NODE() records the node's own address, + * which sort_nonatomic() invalidates by relocating the entries. + */ + for (i = 0; i < fwspec->num_ids; i++) + RB_CLEAR_NODE(&master->streams[i].node); + mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) { struct arm_smmu_stream *new_stream = &master->streams[i]; @@ -4352,7 +4359,9 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, if (ret) { for (i--; i >= 0; i--) - rb_erase(&master->streams[i].node, &smmu->streams); + if (!RB_EMPTY_NODE(&master->streams[i].node)) + rb_erase(&master->streams[i].node, + &smmu->streams); kfree(master->streams); kfree(master->build_invs); } @@ -4372,7 +4381,8 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) - rb_erase(&master->streams[i].node, &smmu->streams); + if (!RB_EMPTY_NODE(&master->streams[i].node)) + rb_erase(&master->streams[i].node, &smmu->streams); mutex_unlock(&smmu->streams_mutex); kfree(master->streams); From 1ae140d0f27248fc588f2f3baa9940512ee078f1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 28 Sep 2026 12:51:47 +0100 Subject: [PATCH 57/69] iommu: Make iommu-dma helpers private Most of iommu-dma's own internals were privatised a while ago, but there are still a couple of DMA-domain-specific helpers provided by the core code. These now definitely have no reason to be exposed publicly, so move their declarations into the private header. Signed-off-by: Robin Murphy Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 1 + drivers/iommu/iommu-priv.h | 4 ++++ include/linux/iommu.h | 2 -- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 58c624513cd4..1b8012123232 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -37,6 +37,7 @@ #include "dma-iommu.h" #include "iommu-pages.h" +#include "iommu-priv.h" struct iommu_dma_msi_page { struct list_head list; diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h index aaffad5854fc..1f9f886b2c10 100644 --- a/drivers/iommu/iommu-priv.h +++ b/drivers/iommu/iommu-priv.h @@ -30,6 +30,10 @@ static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwsp void iommu_fwspec_free(struct device *dev); +int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); + +struct iommu_domain *iommu_get_dma_domain(struct device *dev); + int iommu_device_register_bus(struct iommu_device *iommu, const struct iommu_ops *ops, const struct bus_type *bus, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index ac43b8b93f14..95e32c78dcb7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -898,7 +898,6 @@ int iommu_device_sysfs_add(struct iommu_device *iommu, void iommu_device_sysfs_remove(struct iommu_device *iommu); int iommu_device_link(struct iommu_device *iommu, struct device *link); void iommu_device_unlink(struct iommu_device *iommu, struct device *link); -int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); static inline struct iommu_device *dev_to_iommu_device(struct device *dev) { @@ -943,7 +942,6 @@ extern void iommu_detach_device(struct iommu_domain *domain, struct device *dev); extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev); -extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); extern int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp); int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, From 1d87b20d1c272daf3bb36a155487fe8cc1da4490 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Wed, 30 Sep 2026 22:14:05 +0800 Subject: [PATCH 58/69] hyperv: Introduce new hypercall interfaces used by Hyper-V guest IOMMU Hyper-V guest IOMMU is a para-virtualized IOMMU based on hypercalls. Introduce the hypercalls used by the child partition to interact with this facility. These hypercalls fall into below categories: - Detection and capability: HVCALL_GET_IOMMU_CAPABILITIES is used to detect the existence and capabilities of the guest IOMMU. - Device management: HVCALL_GET_LOGICAL_DEVICE_PROPERTY is used to check whether an endpoint device is managed by the guest IOMMU. - Domain management: A set of hypercalls is provided to handle the creation, configuration, and deletion of guest domains, as well as the attachment/detachment of endpoint devices to/from those domains. - IOTLB flushing: HVCALL_FLUSH_DEVICE_DOMAIN is used to ask Hyper-V for a domain-selective IOTLB flush (which in its handler may flush the device TLB as well). Signed-off-by: Wei Liu Co-developed-by: Easwar Hariharan Signed-off-by: Easwar Hariharan Co-developed-by: Yu Zhang Signed-off-by: Yu Zhang Reviewed-by: Jacob Pan Reviewed-by: Michael Kelley Signed-off-by: Joerg Roedel --- drivers/hv/hv_common.c | 1 + include/hyperv/hvgdk_mini.h | 9 +++ include/hyperv/hvhdk_mini.h | 137 ++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 31256cb22b39..8896fb4809b5 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -808,6 +808,7 @@ static const struct hv_status_info hv_status_infos[] = { _STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EOPNOTSUPP), _STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL), _STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL), + _STATUS_INFO(HV_STATUS_DEVICE_ALREADY_IN_DOMAIN, -EEXIST), _STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO), _STATUS_INFO(HV_STATUS_TIME_OUT, -EIO), _STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO), diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index 6a4e8b9d570f..5d1338197eee 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h @@ -37,6 +37,7 @@ struct hv_u128 { #define HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED 0x20 #define HV_STATUS_INVALID_LP_INDEX 0x41 #define HV_STATUS_INVALID_REGISTER_VALUE 0x50 +#define HV_STATUS_DEVICE_ALREADY_IN_DOMAIN 0x66 #define HV_STATUS_OPERATION_FAILED 0x71 #define HV_STATUS_INSUFFICIENT_ROOT_MEMORY 0x73 #define HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY 0x75 @@ -486,10 +487,16 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */ #define HVCALL_GET_VP_INDEX_FROM_APIC_ID 0x009a #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 +#define HVCALL_CREATE_DEVICE_DOMAIN 0x00b1 +#define HVCALL_ATTACH_DEVICE_DOMAIN 0x00b2 #define HVCALL_SIGNAL_EVENT_DIRECT 0x00c0 #define HVCALL_POST_MESSAGE_DIRECT 0x00c1 #define HVCALL_DISPATCH_VP 0x00c2 +#define HVCALL_DETACH_DEVICE_DOMAIN 0x00c4 +#define HVCALL_DELETE_DEVICE_DOMAIN 0x00c5 #define HVCALL_GET_GPA_PAGES_ACCESS_STATES 0x00c9 +#define HVCALL_CONFIGURE_DEVICE_DOMAIN 0x00ce +#define HVCALL_FLUSH_DEVICE_DOMAIN 0x00d0 #define HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d7 #define HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d8 #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db @@ -502,6 +509,8 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */ #define HVCALL_MMIO_READ 0x0106 #define HVCALL_MMIO_WRITE 0x0107 #define HVCALL_DISABLE_HYP_EX 0x010f +#define HVCALL_GET_IOMMU_CAPABILITIES 0x0125 +#define HVCALL_GET_LOGICAL_DEVICE_PROPERTY 0x0127 #define HVCALL_MAP_STATS_PAGE2 0x0131 /* HV_HYPERCALL_INPUT */ diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h index 035ba20870f7..7b9f43b320cd 100644 --- a/include/hyperv/hvhdk_mini.h +++ b/include/hyperv/hvhdk_mini.h @@ -548,4 +548,141 @@ union hv_device_id { /* HV_DEVICE_ID */ } acpi; } __packed; +/* Device domain types */ +#define HV_DEVICE_DOMAIN_TYPE_S1 1 /* HV_DEVICE_DOMAIN_ID_TYPE_S1 */ + +/* ID for default domain and NULL domain */ +#define HV_DEVICE_DOMAIN_ID_DEFAULT 0 +#define HV_DEVICE_DOMAIN_ID_NULL 0xFFFFFFFFULL + +union hv_device_domain_id { /* HV_DEVICE_DOMAIN_ID */ + u64 as_uint64; + struct { + u32 type: 4; + u32 reserved: 28; + u32 id; + } __packed; +}; + +struct hv_input_device_domain { /* HV_INPUT_DEVICE_DOMAIN */ + u64 partition_id; + union hv_input_vtl owner_vtl; + u8 padding[7]; + union hv_device_domain_id domain_id; +} __packed; + +/* HV_CREATE_DEVICE_DOMAIN_FLAGS */ +union hv_create_device_domain_flags { + u32 as_uint32; + struct { + u32 forward_progress_required: 1; + u32 inherit_owning_vtl: 1; + u32 reserved: 30; + } __packed; +}; + +/* HV_INPUT_CREATE_DEVICE_DOMAIN */ +struct hv_input_create_device_domain { + struct hv_input_device_domain device_domain; + union hv_create_device_domain_flags create_device_domain_flags; + u32 padding; +} __packed; + +/* HV_INPUT_DELETE_DEVICE_DOMAIN */ +struct hv_input_delete_device_domain { + struct hv_input_device_domain device_domain; +} __packed; + +/* HV_INPUT_ATTACH_DEVICE_DOMAIN */ +struct hv_input_attach_device_domain { + struct hv_input_device_domain device_domain; + union hv_device_id device_id; +} __packed; + +/* HV_INPUT_DETACH_DEVICE_DOMAIN */ +struct hv_input_detach_device_domain { + u64 partition_id; + union hv_device_id device_id; +} __packed; + +struct hv_device_domain_settings { /* HV_DEVICE_DOMAIN_SETTINGS */ + struct { + /* + * Enable translations. If not enabled, all transaction bypass + * S1 translations. + */ + u64 translation_enabled: 1; + u64 blocked: 1; + /* + * First stage address translation paging mode: + * 0: 4-level paging (default) + * 1: 5-level paging + */ + u64 first_stage_paging_mode: 1; + u64 reserved: 61; + } flags; + + /* Address of translation table */ + u64 page_table_root; +} __packed; + +/* HV_INPUT_CONFIGURE_DEVICE_DOMAIN */ +struct hv_input_configure_device_domain { + struct hv_input_device_domain device_domain; + struct hv_device_domain_settings settings; +} __packed; + +/* HV_INPUT_GET_IOMMU_CAPABILITIES */ +struct hv_input_get_iommu_capabilities { + u64 partition_id; + u64 reserved; +} __packed; + +/* HV_OUTPUT_GET_IOMMU_CAPABILITIES */ +struct hv_output_get_iommu_capabilities { + u32 size; + u16 reserved; + u8 max_iova_width; + u8 max_pasid_width; + +#define HV_IOMMU_CAP_PRESENT BIT_ULL(0) +#define HV_IOMMU_CAP_S2 BIT_ULL(1) +#define HV_IOMMU_CAP_S1 BIT_ULL(2) +#define HV_IOMMU_CAP_S1_5LVL BIT_ULL(3) +#define HV_IOMMU_CAP_PASID BIT_ULL(4) +#define HV_IOMMU_CAP_ATS BIT_ULL(5) +#define HV_IOMMU_CAP_PRI BIT_ULL(6) + + u64 iommu_cap; + u64 pgsize_bitmap; +} __packed; + +/* HV_LOGICAL_DEVICE_PROPERTY_CODE */ +enum hv_logical_device_property_code { + HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU = 10, +}; + +/* HV_INPUT_GET_LOGICAL_DEVICE_PROPERTY */ +struct hv_input_get_logical_device_property { + u64 partition_id; + u64 logical_device_id; + /* Takes values from enum hv_logical_device_property_code. */ + u32 code; + u32 reserved; +} __packed; + +/* HV_OUTPUT_GET_LOGICAL_DEVICE_PROPERTY */ +struct hv_output_get_logical_device_property { +#define HV_DEVICE_IOMMU_ENABLED BIT_ULL(0) + u64 device_iommu; + u64 reserved; +} __packed; + +/* HV_INPUT_FLUSH_DEVICE_DOMAIN */ +struct hv_input_flush_device_domain { + struct hv_input_device_domain device_domain; + u32 flags; + u32 reserved; +} __packed; + #endif /* _HV_HVHDK_MINI_H */ From 118bdac1054628c04c547c2c988bb06ca43677a3 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Sep 2026 22:14:06 +0800 Subject: [PATCH 59/69] Drivers: hv: Add logical device ID registry for vPCI devices Hyper-V identifies each PCI pass-thru device by a logical device ID in its hypercall interface. This ID consists of a per-bus prefix, derived from the VMBus device instance GUID, combined with the PCI function number of the endpoint device. Add a registry in hv_common.c that maps a PCI domain number to its logical device ID prefix. The vPCI bus driver (pci-hyperv) registers the prefix when a bus is probed and unregisters it when the bus is removed. Consumers such as the para-virtualized IOMMU driver look up the prefix by PCI domain number and combine it with the function number to form the complete logical device ID for hypercalls. Use rhashtable for the sparse exact-match mapping. Lookups copy the prefix while holding the RCU read lock, and removal defers freeing the entry until existing readers have completed. The prefix construction is shared via hv_build_logical_dev_id_prefix() so that pci-hyperv's interrupt retargeting path and the registry use exactly the same byte layout. It is derived on demand from the constant hv_device instance GUID rather than cached in struct hv_pcibus_device, which is private to the pci-hyperv module; this keeps the interface narrow and avoids depending on pci-hyperv internals. Co-developed-by: Easwar Hariharan Signed-off-by: Easwar Hariharan Signed-off-by: Yu Zhang Reviewed-by: Michael Kelley Signed-off-by: Joerg Roedel --- drivers/hv/hv_common.c | 114 ++++++++++++++++++++++++++++ drivers/pci/controller/pci-hyperv.c | 21 +++-- include/asm-generic/mshyperv.h | 4 + include/linux/hyperv.h | 8 ++ 4 files changed, 142 insertions(+), 5 deletions(-) diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 8896fb4809b5..4f3bb9b4e87c 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -78,6 +79,22 @@ static struct ctl_table_header *hv_ctl_table_hdr; u8 * __percpu *hv_synic_eventring_tail; EXPORT_SYMBOL_GPL(hv_synic_eventring_tail); +struct hv_pci_busdata { + int pci_domain_nr; + u32 logical_dev_id_prefix; + struct rhash_head node; + struct rcu_head rcu; +}; + +static struct rhashtable hv_pci_bus_ht; +static bool hv_pci_bus_ht_initialized; + +static const struct rhashtable_params hv_pci_bus_ht_params = { + .key_len = sizeof_field(struct hv_pci_busdata, pci_domain_nr), + .key_offset = offsetof(struct hv_pci_busdata, pci_domain_nr), + .head_offset = offsetof(struct hv_pci_busdata, node), +}; + /* * Hyper-V specific initialization and shutdown code that is * common across all architectures. Called from architecture @@ -86,6 +103,11 @@ EXPORT_SYMBOL_GPL(hv_synic_eventring_tail); void __init hv_common_free(void) { + if (hv_pci_bus_ht_initialized) { + rhashtable_destroy(&hv_pci_bus_ht); + hv_pci_bus_ht_initialized = false; + } + unregister_sysctl_table(hv_ctl_table_hdr); hv_ctl_table_hdr = NULL; @@ -315,6 +337,7 @@ u8 __init get_vtl(void) int __init hv_common_init(void) { int i; + int ret; union hv_hypervisor_version_info version; /* Get information about the Microsoft Hypervisor version */ @@ -394,6 +417,13 @@ int __init hv_common_init(void) for (i = 0; i < nr_cpu_ids; i++) hv_vp_index[i] = VP_INVAL; + ret = rhashtable_init(&hv_pci_bus_ht, &hv_pci_bus_ht_params); + if (ret) { + hv_common_free(); + return ret; + } + hv_pci_bus_ht_initialized = true; + return 0; } @@ -862,3 +892,87 @@ const char *hv_result_to_string(u64 status) return "Unknown"; } EXPORT_SYMBOL_GPL(hv_result_to_string); + +/* + * Logical device ID registry for Hyper-V PCI buses. The pci-hyperv + * driver registers each bus's logical device ID prefix before scanning + * its devices. Consumers look up the prefix by PCI domain number when + * building logical device IDs for Hyper-V interfaces. + */ +int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix) +{ + struct hv_pci_busdata *bus, *new; + int ret; + + new = kzalloc_obj(*new, GFP_KERNEL); + if (!new) + return -ENOMEM; + + new->pci_domain_nr = pci_domain_nr; + new->logical_dev_id_prefix = logical_dev_id_prefix; + + bus = rhashtable_lookup_get_insert_fast(&hv_pci_bus_ht, &new->node, + hv_pci_bus_ht_params); + if (IS_ERR(bus)) { + ret = PTR_ERR(bus); + goto free_new; + } + + if (WARN_ONCE(bus != NULL, + "Hyper-V PCI domain %04x is already registered\n", + pci_domain_nr)) { + ret = -EEXIST; + goto free_new; + } + + return 0; + +free_new: + kfree(new); + return ret; +} +EXPORT_SYMBOL_FOR_MODULES(hv_pci_register_dev_id, "pci-hyperv"); + +void hv_pci_unregister_dev_id(int pci_domain_nr) +{ + struct hv_pci_busdata *bus; + int ret = -ENOENT; + + rcu_read_lock(); + bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr, + hv_pci_bus_ht_params); + if (bus) + ret = rhashtable_remove_fast(&hv_pci_bus_ht, &bus->node, + hv_pci_bus_ht_params); + rcu_read_unlock(); + + if (WARN_ONCE(ret, + "Failed to unregister Hyper-V PCI domain %04x\n", + pci_domain_nr)) + return; + + kfree_rcu(bus, rcu); +} +EXPORT_SYMBOL_FOR_MODULES(hv_pci_unregister_dev_id, "pci-hyperv"); + +/* + * Look up the logical device ID prefix registered for @pci_domain_nr. + * Returns 0 on success with *prefix filled in; -ENODEV if no entry is + * registered for that PCI domain. + */ +int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix) +{ + struct hv_pci_busdata *bus; + int ret = -ENODEV; + + rcu_read_lock(); + bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr, + hv_pci_bus_ht_params); + if (bus) { + *prefix = bus->logical_dev_id_prefix; + ret = 0; + } + rcu_read_unlock(); + + return ret; +} diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 89816a2bd7cd..db70be47af6e 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -641,10 +641,7 @@ static void hv_irq_retarget_interrupt(struct irq_data *data) params->int_entry.source = HV_INTERRUPT_SOURCE_MSI; params->int_entry.msi_entry.address.as_uint32 = int_desc->address & 0xffffffff; params->int_entry.msi_entry.data.as_uint32 = int_desc->data; - params->device_id = (hbus->hdev->dev_instance.b[5] << 24) | - (hbus->hdev->dev_instance.b[4] << 16) | - (hbus->hdev->dev_instance.b[7] << 8) | - (hbus->hdev->dev_instance.b[6] & 0xf8) | + params->device_id = hv_build_logical_dev_id_prefix(hbus->hdev) | PCI_FUNC(pdev->devfn); params->int_target.vector = hv_msi_get_int_vector(data); @@ -3716,6 +3713,7 @@ static int hv_pci_probe(struct hv_device *hdev, struct hv_pcibus_device *hbus; int ret, dom; u16 dom_req; + u32 prefix; char *name; bridge = devm_pci_alloc_host_bridge(&hdev->device, 0); @@ -3858,13 +3856,22 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->state = hv_pcibus_probed; - ret = create_root_hv_pci_bus(hbus); + /* Register the bus before scanning any devices on it. */ + prefix = hv_build_logical_dev_id_prefix(hdev); + + ret = hv_pci_register_dev_id(dom, prefix); if (ret) goto free_windows; + ret = create_root_hv_pci_bus(hbus); + if (ret) + goto unregister_pci_bus; + mutex_unlock(&hbus->state_lock); return 0; +unregister_pci_bus: + hv_pci_unregister_dev_id(dom); free_windows: hv_pci_free_bridge_windows(hbus); exit_d0: @@ -3978,6 +3985,8 @@ static void hv_pci_remove(struct hv_device *hdev) hbus = hv_get_drvdata(hdev); if (hbus->state == hv_pcibus_installed) { + int dom = hbus->bridge->domain_nr; + tasklet_disable(&hdev->channel->callback_event); hbus->state = hv_pcibus_removing; tasklet_enable(&hdev->channel->callback_event); @@ -3995,6 +4004,8 @@ static void hv_pci_remove(struct hv_device *hdev) hv_pci_remove_slots(hbus); pci_remove_root_bus(hbus->bridge->bus); pci_unlock_rescan_remove(); + + hv_pci_unregister_dev_id(dom); } hv_pci_bus_exit(hdev, false); diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index bf601d67cecb..34d4c332449c 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -73,6 +73,10 @@ extern enum hv_partition_type hv_curr_partition_type; extern void * __percpu *hyperv_pcpu_input_arg; extern void * __percpu *hyperv_pcpu_output_arg; +int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix); +void hv_pci_unregister_dev_id(int pci_domain_nr); +int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix); + u64 hv_do_hypercall(u64 control, void *inputaddr, void *outputaddr); u64 hv_do_fast_hypercall8(u16 control, u64 input8); u64 hv_do_fast_hypercall16(u16 control, u64 input1, u64 input2); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9e109d91aa14..e61f9a4cb7c3 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1299,6 +1299,14 @@ struct hv_device { #define device_to_hv_device(d) container_of_const(d, struct hv_device, device) #define drv_to_hv_drv(d) container_of_const(d, struct hv_driver, driver) +static inline u32 hv_build_logical_dev_id_prefix(struct hv_device *hdev) +{ + return ((u32)hdev->dev_instance.b[5] << 24) | + ((u32)hdev->dev_instance.b[4] << 16) | + ((u32)hdev->dev_instance.b[7] << 8) | + (hdev->dev_instance.b[6] & 0xf8u); +} + static inline void hv_set_drvdata(struct hv_device *dev, void *data) { dev_set_drvdata(&dev->device, data); From 07f8807ca978f486243e93aa4e85d20df871cd7f Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Sep 2026 22:14:07 +0800 Subject: [PATCH 60/69] iommu/x86: Add architectural MSI reserved region helper Intel and AMD IOMMU drivers both reserve the x86 architectural MSI address window from normal IOVA mappings, but each open-codes the range, allocation attributes, and insertion into the reserved-region list. Add iommu_add_resv_x86_msi_region() to encapsulate those operations, and convert the Intel and AMD drivers to use it. Return an error on allocation failure so callers can preserve their existing error paths. Signed-off-by: Yu Zhang Reviewed-by: Michael Kelley Reviewed-by: Jason Gunthorpe Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- arch/x86/include/asm/iommu.h | 20 ++++++++++++++++++++ drivers/iommu/amd/iommu.c | 8 +------- drivers/iommu/intel/iommu.c | 11 ++--------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h index 3be2451e7bc8..4e1e55c3c09a 100644 --- a/arch/x86/include/asm/iommu.h +++ b/arch/x86/include/asm/iommu.h @@ -4,6 +4,11 @@ #include +#ifdef CONFIG_IOMMU_API +#include +#include +#endif + #include extern int force_iommu, no_iommu; @@ -21,6 +26,21 @@ extern bool x86_swiotlb_enable; /* 10 seconds */ #define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) +#ifdef CONFIG_IOMMU_API +static inline int iommu_add_resv_x86_msi_region(struct list_head *head) +{ + struct iommu_resv_region *region; + + region = iommu_alloc_resv_region(0xfee00000, SZ_1M, 0, + IOMMU_RESV_MSI, GFP_KERNEL); + if (!region) + return -ENOMEM; + + list_add_tail(®ion->list, head); + return 0; +} +#endif + static inline int __init arch_rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr) { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 56262f6b1f70..391cec4d86c0 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -50,8 +50,6 @@ #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) /* Reserved IOVA ranges */ -#define MSI_RANGE_START (0xfee00000) -#define MSI_RANGE_END (0xfeefffff) #define HT_RANGE_START (0xfd00000000ULL) #define HT_RANGE_END (0xffffffffffULL) @@ -3137,12 +3135,8 @@ static void amd_iommu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } - region = iommu_alloc_resv_region(MSI_RANGE_START, - MSI_RANGE_END - MSI_RANGE_START + 1, - 0, IOMMU_RESV_MSI, GFP_KERNEL); - if (!region) + if (iommu_add_resv_x86_msi_region(head)) return; - list_add_tail(®ion->list, head); if (amd_iommu_ht_range_ignore()) return; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 2e3b3ab216f8..74fb5c894fe5 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -39,8 +39,6 @@ #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e) -#define IOAPIC_RANGE_START (0xfee00000) -#define IOAPIC_RANGE_END (0xfeefffff) #define IOVA_START_ADDR (0x1000) #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57 @@ -3412,7 +3410,6 @@ static void intel_iommu_get_resv_regions(struct device *device, struct list_head *head) { int prot = DMA_PTE_READ | DMA_PTE_WRITE; - struct iommu_resv_region *reg; struct dmar_rmrr_unit *rmrr; struct device *i_dev; int i; @@ -3448,6 +3445,7 @@ static void intel_iommu_get_resv_regions(struct device *device, #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA if (dev_is_pci(device)) { struct pci_dev *pdev = to_pci_dev(device); + struct iommu_resv_region *reg; if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) { reg = iommu_alloc_resv_region(0, 1UL << 24, prot, @@ -3459,12 +3457,7 @@ static void intel_iommu_get_resv_regions(struct device *device, } #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */ - reg = iommu_alloc_resv_region(IOAPIC_RANGE_START, - IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1, - 0, IOMMU_RESV_MSI, GFP_KERNEL); - if (!reg) - return; - list_add_tail(®->list, head); + iommu_add_resv_x86_msi_region(head); } static struct iommu_group *intel_iommu_device_group(struct device *dev) From 2690d4156b51f7e9fe874dd480fc928594615552 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Sep 2026 22:14:08 +0800 Subject: [PATCH 61/69] iommu/hyperv: Add para-virtualized IOMMU support for Hyper-V guest Add a para-virtualized IOMMU driver for Linux guests running on Hyper-V. This driver implements stage-1 IO translation within the guest OS. It integrates with the Linux IOMMU core, utilizing Hyper-V hypercalls for: - Capability discovery - Domain allocation, configuration, and deallocation - Device attachment and detachment - IOTLB invalidation The driver constructs x86-compatible stage-1 IO page tables in the guest memory using consolidated IO page table helpers. This allows the guest to manage stage-1 translations independently of vendor- specific drivers (like Intel VT-d or AMD IOMMU). Hyper-V consumes this stage-1 IO page table when a device domain is created and configured, and nests it with the host's stage-2 IO page tables, therefore eliminating the VM exits for guest IOMMU mapping operations. For unmapping operations, VM exits to perform the IOTLB flush are still unavoidable. Guest hibernation and resume are not supported by this initial implementation. The guest-owned stage-1 page tables reside in guest memory and are preserved in the hibernation image. However, the image does not by itself preserve the Hyper-V device-domain objects or their device attachments, and the current pvIOMMU interface does not define whether or how Hyper-V restores that state. To identify a device in its hypercall interface, the driver looks up the logical device ID prefix registered for the device's PCI domain (see the logical device ID registry in hv_common.c) and combines it with the PCI function number of the endpoint device. Co-developed-by: Wei Liu Signed-off-by: Wei Liu Co-developed-by: Easwar Hariharan Signed-off-by: Easwar Hariharan Signed-off-by: Yu Zhang Reviewed-by: Jacob Pan Reviewed-by: Michael Kelley Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- arch/x86/hyperv/hv_init.c | 4 + arch/x86/include/asm/mshyperv.h | 4 + drivers/iommu/Kconfig | 1 + drivers/iommu/hyperv/Kconfig | 15 + drivers/iommu/hyperv/Makefile | 1 + drivers/iommu/hyperv/hv-iommu-guest.c | 610 ++++++++++++++++++++++++++ drivers/iommu/hyperv/iommu.h | 44 ++ 7 files changed, 679 insertions(+) create mode 100644 drivers/iommu/hyperv/Kconfig create mode 100644 drivers/iommu/hyperv/hv-iommu-guest.c create mode 100644 drivers/iommu/hyperv/iommu.h diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 0b4a1c0b0b16..d5edc8530964 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -579,6 +579,10 @@ skip_hypercall_pg_init: old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev; x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev; +#ifdef CONFIG_HYPERV_PVIOMMU + x86_init.iommu.iommu_init = hv_iommu_init; +#endif + hv_apic_init(); x86_init.pci.arch_init = hv_pci_init; diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index f64393e853ee..20d947c2c758 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -313,6 +313,10 @@ static inline void mshv_vtl_return_hypercall(void) {} static inline void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0) {} #endif +#ifdef CONFIG_HYPERV_PVIOMMU +int __init hv_iommu_init(void); +#endif + #include #endif diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd69467a..0d128f377929 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -195,6 +195,7 @@ config MSM_IOMMU source "drivers/iommu/amd/Kconfig" source "drivers/iommu/arm/Kconfig" source "drivers/iommu/intel/Kconfig" +source "drivers/iommu/hyperv/Kconfig" source "drivers/iommu/iommufd/Kconfig" source "drivers/iommu/riscv/Kconfig" diff --git a/drivers/iommu/hyperv/Kconfig b/drivers/iommu/hyperv/Kconfig new file mode 100644 index 000000000000..fe24ed65b6a7 --- /dev/null +++ b/drivers/iommu/hyperv/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Hyper-V para-virtualized IOMMU driver for Linux guests +config HYPERV_PVIOMMU + bool "Microsoft Hyper-V para-virtualized IOMMU driver for Linux guests" + depends on X86_64 && HYPERV + select IOMMU_DMA + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_X86_64 + default HYPERV + help + Para-virtualized IOMMU driver for Linux guests running on + Microsoft Hyper-V. Provides DMA remapping and IOTLB + flush support to enable DMA isolation for PCI devices + assigned to the guest. diff --git a/drivers/iommu/hyperv/Makefile b/drivers/iommu/hyperv/Makefile index 6ef0ef97f3dd..d3b2a259439c 100644 --- a/drivers/iommu/hyperv/Makefile +++ b/drivers/iommu/hyperv/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_IRQ_REMAP) += hv-irq-remap-x86.o +obj-$(CONFIG_HYPERV_PVIOMMU) += hv-iommu-guest.o diff --git a/drivers/iommu/hyperv/hv-iommu-guest.c b/drivers/iommu/hyperv/hv-iommu-guest.c new file mode 100644 index 000000000000..341d70eed592 --- /dev/null +++ b/drivers/iommu/hyperv/hv-iommu-guest.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Hyper-V para-virtualized IOMMU driver for Linux guests. + * + * Copyright (C) 2024-2026 Microsoft, Inc. + */ + +#define pr_fmt(fmt) "Hyper-V pvIOMMU: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "iommu.h" +#include "../iommu-pages.h" + +struct hv_iommu_dev *hv_iommu_device; + +/* + * Identity and blocking domains are static singletons: identity is a 1:1 + * passthrough with no page table, blocking rejects all DMA. Neither holds + * per-IOMMU state, so one instance suffices even with multiple vIOMMUs. + */ +static const struct iommu_domain_ops hv_iommu_identity_domain_ops; +static const struct iommu_domain_ops hv_iommu_blocking_domain_ops; +static struct iommu_ops hv_iommu_ops; + +static struct hv_iommu_domain hv_identity_domain = { + .domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &hv_iommu_identity_domain_ops, + .owner = &hv_iommu_ops, + }, +}; + +static struct hv_iommu_domain hv_blocking_domain = { + .domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &hv_iommu_blocking_domain_ops, + .owner = &hv_iommu_ops, + }, +}; + +static inline bool hv_iommu_present(u64 cap) +{ + return cap & HV_IOMMU_CAP_PRESENT; +} + +static inline bool hv_iommu_s1_domain_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_S1; +} + +static inline bool hv_iommu_5lvl_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_S1_5LVL; +} + +static inline bool hv_iommu_ats_supported(u64 cap) +{ + return cap & HV_IOMMU_CAP_ATS; +} + +static int hv_create_device_domain(struct hv_iommu_domain *hv_domain) +{ + int ret; + u64 status; + unsigned long flags; + struct hv_input_create_device_domain *input; + + ret = ida_alloc_range(&hv_iommu_device->domain_ids, + hv_iommu_device->first_domain, + hv_iommu_device->last_domain, GFP_KERNEL); + if (ret < 0) + return ret; + + hv_domain->device_domain.partition_id = HV_PARTITION_ID_SELF; + hv_domain->device_domain.domain_id.type = HV_DEVICE_DOMAIN_TYPE_S1; + hv_domain->device_domain.domain_id.id = ret; + hv_domain->hv_iommu = hv_iommu_device; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->create_device_domain_flags.forward_progress_required = 1; + input->create_device_domain_flags.inherit_owning_vtl = 0; + status = hv_do_hypercall(HVCALL_CREATE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) { + hv_status_err(status, "HVCALL_CREATE_DEVICE_DOMAIN failed\n"); + ida_free(&hv_iommu_device->domain_ids, + hv_domain->device_domain.domain_id.id); + } + + return hv_result_to_errno(status); +} + +static void hv_delete_device_domain(struct hv_iommu_domain *hv_domain) +{ + u64 status; + unsigned long flags; + struct hv_input_delete_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_DELETE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, "HVCALL_DELETE_DEVICE_DOMAIN failed\n"); + + ida_free(&hv_domain->hv_iommu->domain_ids, + hv_domain->device_domain.domain_id.id); +} + +static int +hv_configure_device_domain(struct hv_iommu_domain *hv_domain, + const struct hv_device_domain_settings *settings) +{ + u64 status; + unsigned long flags; + struct hv_input_configure_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->settings = *settings; + status = hv_do_hypercall(HVCALL_CONFIGURE_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, + "HVCALL_CONFIGURE_DEVICE_DOMAIN failed\n"); + + return hv_result_to_errno(status); +} + +static int +hv_create_configure_device_domain(struct hv_iommu_domain *hv_domain, + const struct hv_device_domain_settings *settings) +{ + int ret; + + ret = hv_create_device_domain(hv_domain); + if (ret) + return ret; + + ret = hv_configure_device_domain(hv_domain, settings); + if (ret) + hv_delete_device_domain(hv_domain); + + return ret; +} + +static bool hv_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + return cap == IOMMU_CAP_CACHE_COHERENCY; +} + +static void hv_flush_device_domain(struct hv_iommu_domain *hv_domain) +{ + u64 status; + unsigned long flags; + struct hv_input_flush_device_domain *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + WARN_ONCE(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN failed: %#llx (%s)\n", + status, hv_result_to_string(status)); +} + +static int hv_iommu_attach_dev(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + u64 status; + u32 prefix; + unsigned long flags; + struct pci_dev *pdev; + struct hv_input_attach_device_domain *input; + struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain); + int pci_domain; + int ret; + + pdev = to_pci_dev(dev); + dev_dbg(dev, "attaching to domain %d\n", + hv_domain->device_domain.domain_id.id); + + pci_domain = pci_domain_nr(pdev->bus); + ret = hv_pci_lookup_dev_id(pci_domain, &prefix); + if (ret) { + dev_err(dev, + "no logical device ID registered for PCI domain %04x\n", + pci_domain); + return ret; + } + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->device_domain = hv_domain->device_domain; + input->device_id.as_uint64 = (u64)prefix | PCI_FUNC(pdev->devfn); + status = hv_do_hypercall(HVCALL_ATTACH_DEVICE_DOMAIN, input, NULL); + + local_irq_restore(flags); + + if (!hv_result_success(status) && + hv_result(status) != HV_STATUS_DEVICE_ALREADY_IN_DOMAIN) { + hv_status_err(status, "HVCALL_ATTACH_DEVICE_DOMAIN failed\n"); + return hv_result_to_errno(status); + } + + if (domain != &hv_blocking_domain.domain && + !pdev->ats_enabled && + hv_iommu_ats_supported(hv_domain->hv_iommu->cap) && + pci_ats_supported(pdev)) + pci_enable_ats(pdev, PAGE_SHIFT); + + return 0; +} + +static int hv_iommu_blocking_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + if (pdev->ats_enabled) + pci_disable_ats(pdev); + + ret = hv_iommu_attach_dev(domain, dev, old); + + /* + * Attaching to the blocking domain only asks the hypervisor to + * disable translation and IOPF for the device, so it cannot fail + * unless there is a driver or hypervisor bug. Return the hypercall + * status rather than 0 so that a failure on the DMA ownership claim + * path (VFIO/iommufd) fails the claim instead of leaving the device + * unblocked. WARN since such a failure indicates a bug. + */ + WARN_ON(ret); + return ret; +} + +static int +hv_iommu_get_dev_property(struct device *dev, u32 code, + struct hv_output_get_logical_device_property *prop) +{ + u64 status; + u32 prefix; + unsigned long flags; + int ret; + struct pci_dev *pdev = to_pci_dev(dev); + struct hv_input_get_logical_device_property *input; + struct hv_output_get_logical_device_property *output; + + ret = hv_pci_lookup_dev_id(pci_domain_nr(pdev->bus), &prefix); + if (ret) + return ret; + + local_irq_save(flags); + + /* + * The combined input and output fit in the per-CPU input page, + * so no separate output page is needed. + */ + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = (struct hv_output_get_logical_device_property *)(input + 1); + memset(input, 0, sizeof(*input)); + input->partition_id = HV_PARTITION_ID_SELF; + input->logical_device_id = (u64)prefix | PCI_FUNC(pdev->devfn); + input->code = code; + status = hv_do_hypercall(HVCALL_GET_LOGICAL_DEVICE_PROPERTY, + input, output); + *prop = *output; + + local_irq_restore(flags); + + if (!hv_result_success(status)) + hv_status_err(status, + "HVCALL_GET_LOGICAL_DEVICE_PROPERTY failed\n"); + + return hv_result_to_errno(status); +} + +static struct iommu_device *hv_iommu_probe_device(struct device *dev) +{ + struct hv_output_get_logical_device_property property = {0}; + int ret; + + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + ret = hv_iommu_get_dev_property(dev, HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU, + &property); + if (ret || !(property.device_iommu & HV_DEVICE_IOMMU_ENABLED)) + return ERR_PTR(-ENODEV); + + return &hv_iommu_device->iommu; +} + +static struct iommu_group *hv_iommu_device_group(struct device *dev) +{ + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + return pci_device_group(dev); +} + +static int __init hv_initialize_static_domains(void) +{ + /* + * Clearing translation_enabled bypasses stage-1 translation, so DMA + * addresses are used directly as GPAs. Hyper-V requires paging and + * blocked domains to keep translation_enabled set. + */ + const struct hv_device_domain_settings identity_settings = { + .flags.translation_enabled = 0, + }; + const struct hv_device_domain_settings blocked_settings = { + .flags = { + .translation_enabled = 1, + .blocked = 1, + }, + }; + int ret; + + /* Default stage-1 identity domain */ + ret = hv_create_configure_device_domain(&hv_identity_domain, + &identity_settings); + if (ret) + return ret; + + /* Default stage-1 blocked domain */ + ret = hv_create_configure_device_domain(&hv_blocking_domain, + &blocked_settings); + if (ret) + goto delete_identity_domain; + + return 0; + +delete_identity_domain: + hv_delete_device_domain(&hv_identity_domain); + return ret; +} + +static void hv_iommu_get_resv_regions(struct device *dev, + struct list_head *head) +{ + iommu_add_resv_x86_msi_region(head); +} + +static void hv_iommu_flush_iotlb_all(struct iommu_domain *domain) +{ + hv_flush_device_domain(to_hv_iommu_domain(domain)); +} + +static void hv_iommu_iotlb_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + hv_flush_device_domain(to_hv_iommu_domain(domain)); + + iommu_put_pages_list(&iotlb_gather->freelist); +} + +static void hv_iommu_paging_domain_free(struct iommu_domain *domain) +{ + struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain); + + /* Free all remaining mappings */ + pt_iommu_deinit(&hv_domain->pt_iommu); + + hv_delete_device_domain(hv_domain); + + kfree(hv_domain); +} + +static const struct iommu_domain_ops hv_iommu_identity_domain_ops = { + .attach_dev = hv_iommu_attach_dev, +}; + +static const struct iommu_domain_ops hv_iommu_blocking_domain_ops = { + .attach_dev = hv_iommu_blocking_attach_dev, +}; + +static const struct iommu_domain_ops hv_iommu_paging_domain_ops = { + .attach_dev = hv_iommu_attach_dev, + IOMMU_PT_DOMAIN_OPS(x86_64), + .flush_iotlb_all = hv_iommu_flush_iotlb_all, + .iotlb_sync = hv_iommu_iotlb_sync, + .free = hv_iommu_paging_domain_free, +}; + +static struct iommu_domain *hv_iommu_domain_alloc_paging(struct device *dev) +{ + int ret; + struct hv_iommu_domain *hv_domain; + struct pt_iommu_x86_64_hw_info pt_info; + struct pt_iommu_x86_64_cfg cfg = {}; + struct hv_device_domain_settings settings = { + .flags = { + .translation_enabled = 1, + }, + }; + + hv_domain = kzalloc_obj(*hv_domain, GFP_KERNEL); + if (!hv_domain) + return ERR_PTR(-ENOMEM); + + hv_domain->pt_iommu.nid = dev_to_node(dev); + + cfg.common.hw_max_vasz_lg2 = hv_iommu_device->max_iova_width; + cfg.common.hw_max_oasz_lg2 = 52; + /* + * Hyper-V S1 domains use a 4-level root for IOVA widths up to + * 48 bits. A 5-level root is used only for wider apertures when + * HV_IOMMU_CAP_S1_5LVL is advertised. + */ + cfg.top_level = (hv_iommu_device->max_iova_width > 48) ? 4 : 3; + + ret = pt_iommu_x86_64_init(&hv_domain->pt_iommu_x86_64, &cfg, GFP_KERNEL); + if (ret) + goto err_free; + + /* Constrain to page sizes the hypervisor supports */ + hv_domain->domain.pgsize_bitmap &= hv_iommu_device->pgsize_bitmap; + + hv_domain->domain.ops = &hv_iommu_paging_domain_ops; + + pt_iommu_x86_64_hw_info(&hv_domain->pt_iommu_x86_64, &pt_info); + settings.page_table_root = pt_info.gcr3_pt; + settings.flags.first_stage_paging_mode = pt_info.levels == 5; + + ret = hv_create_configure_device_domain(hv_domain, &settings); + if (ret) + goto err_pt_deinit; + + return &hv_domain->domain; + +err_pt_deinit: + pt_iommu_deinit(&hv_domain->pt_iommu); +err_free: + kfree(hv_domain); + return ERR_PTR(ret); +} + +static struct iommu_ops hv_iommu_ops = { + .capable = hv_iommu_capable, + .domain_alloc_paging = hv_iommu_domain_alloc_paging, + .probe_device = hv_iommu_probe_device, + .device_group = hv_iommu_device_group, + .get_resv_regions = hv_iommu_get_resv_regions, + .owner = THIS_MODULE, + .identity_domain = &hv_identity_domain.domain, + .blocked_domain = &hv_blocking_domain.domain, + .release_domain = &hv_blocking_domain.domain, +}; + +static u64 hv_iommu_detect(struct hv_output_get_iommu_capabilities *cap) +{ + u64 status; + unsigned long flags; + struct hv_input_get_iommu_capabilities *input; + struct hv_output_get_iommu_capabilities *output; + + local_irq_save(flags); + + /* + * The combined input and output fit in the per-CPU input page, + * so no separate output page is needed. + */ + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = (struct hv_output_get_iommu_capabilities *)(input + 1); + memset(input, 0, sizeof(*input)); + input->partition_id = HV_PARTITION_ID_SELF; + status = hv_do_hypercall(HVCALL_GET_IOMMU_CAPABILITIES, input, output); + *cap = *output; + + local_irq_restore(flags); + + return status; +} + +static void __init +hv_init_iommu_device(struct hv_iommu_dev *hv_iommu, + struct hv_output_get_iommu_capabilities *hv_iommu_cap) +{ + ida_init(&hv_iommu->domain_ids); + + hv_iommu->cap = hv_iommu_cap->iommu_cap; + hv_iommu->max_iova_width = hv_iommu_cap->max_iova_width; + if (!hv_iommu_5lvl_supported(hv_iommu->cap) && + hv_iommu->max_iova_width > 48) { + pr_info("5-level paging not supported, limiting iova width to 48.\n"); + hv_iommu->max_iova_width = 48; + } + + hv_iommu->first_domain = HV_DEVICE_DOMAIN_ID_DEFAULT + 1; + hv_iommu->last_domain = HV_DEVICE_DOMAIN_ID_NULL - 1; + hv_iommu->pgsize_bitmap = hv_iommu_cap->pgsize_bitmap; + hv_iommu_device = hv_iommu; +} + +int __init hv_iommu_init(void) +{ + u64 status; + int ret = 0; + struct hv_iommu_dev *hv_iommu = NULL; + struct hv_output_get_iommu_capabilities hv_iommu_cap = {0}; + + if (no_iommu || iommu_detected) + return -ENODEV; + + if (!hv_is_hyperv_initialized()) + return -ENODEV; + + status = hv_iommu_detect(&hv_iommu_cap); + if (!hv_result_success(status)) { + if (hv_result(status) == HV_STATUS_INVALID_HYPERCALL_CODE) + return -ENODEV; + + hv_status_err(status, "HVCALL_GET_IOMMU_CAPABILITIES failed\n"); + return hv_result_to_errno(status); + } + + if (!hv_iommu_present(hv_iommu_cap.iommu_cap)) + return -ENODEV; + + if (!hv_iommu_s1_domain_supported(hv_iommu_cap.iommu_cap)) { + pr_err("stage-1 translation not supported: cap=%#llx\n", + hv_iommu_cap.iommu_cap); + return -ENODEV; + } + + /* + * Require the base page size. The domain page-size bitmap is later + * restricted to the sizes supported by both iommupt and Hyper-V. + */ + if (!(hv_iommu_cap.pgsize_bitmap & PAGE_SIZE)) { + pr_err("unsupported page sizes: pgsize_bitmap=0x%llx\n", + hv_iommu_cap.pgsize_bitmap); + return -ENODEV; + } + + iommu_detected = 1; + pci_request_acs(); + + hv_iommu = kzalloc_obj(*hv_iommu, GFP_KERNEL); + if (!hv_iommu) + return -ENOMEM; + + hv_init_iommu_device(hv_iommu, &hv_iommu_cap); + + ret = hv_initialize_static_domains(); + if (ret) + goto err_free; + + ret = iommu_device_sysfs_add(&hv_iommu->iommu, NULL, NULL, "%s", "hv-iommu"); + if (ret) { + pr_err("iommu_device_sysfs_add failed: %d\n", ret); + goto err_delete_static_domains; + } + + ret = iommu_device_register(&hv_iommu->iommu, &hv_iommu_ops, NULL); + if (ret) { + pr_err("iommu_device_register failed: %d\n", ret); + goto err_sysfs_remove; + } + + pr_info("initialized: %u-bit IOVA aperture, page-size bitmap %#llx\n", + hv_iommu->max_iova_width, hv_iommu->pgsize_bitmap); + return 0; + +err_sysfs_remove: + iommu_device_sysfs_remove(&hv_iommu->iommu); +err_delete_static_domains: + hv_delete_device_domain(&hv_blocking_domain); + hv_delete_device_domain(&hv_identity_domain); +err_free: + kfree(hv_iommu); + return ret; +} diff --git a/drivers/iommu/hyperv/iommu.h b/drivers/iommu/hyperv/iommu.h new file mode 100644 index 000000000000..c412dc1fd037 --- /dev/null +++ b/drivers/iommu/hyperv/iommu.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Hyper-V IOMMU driver. + * + * Copyright (C) 2024-2026, Microsoft, Inc. + * + */ + +#ifndef _HYPERV_IOMMU_H +#define _HYPERV_IOMMU_H + +struct hv_iommu_dev { + struct iommu_device iommu; + struct ida domain_ids; + + /* Device configuration */ + u8 max_iova_width; + u8 max_pasid_width; + u64 cap; + u64 pgsize_bitmap; + + u64 first_domain; + u64 last_domain; +}; + +struct hv_iommu_domain { + union { + struct iommu_domain domain; + struct pt_iommu pt_iommu; + struct pt_iommu_x86_64 pt_iommu_x86_64; + }; + struct hv_iommu_dev *hv_iommu; + struct hv_input_device_domain device_domain; + u64 pgsize_bitmap; +}; + +PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu, domain); +PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu_x86_64.iommu, domain); + +#define to_hv_iommu_domain(d) \ + container_of(d, struct hv_iommu_domain, domain) + +#endif /* _HYPERV_IOMMU_H */ From 87c5e5ae83492d951877eb5711dc03f2314d2695 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Sep 2026 22:14:09 +0800 Subject: [PATCH 62/69] iommu/hyperv: Add page-selective IOTLB flush support Add page-selective IOTLB flush using HVCALL_FLUSH_DEVICE_DOMAIN_LIST. This hypercall accepts a list of (page_number, page_mask_shift) entries, enabling finer-grained IOTLB invalidation compared to the domain-wide HVCALL_FLUSH_DEVICE_DOMAIN used by hv_iommu_flush_iotlb_all(). hv_iommu_calc_flush_range() computes the smallest power-of-two aligned range that covers the target IOVA region, producing a single flush descriptor. This may over-flush when the range is not naturally aligned, matching the approach used by Intel VT-d PSI. If the page-selective flush fails, the code falls back to a full domain flush. Signed-off-by: Easwar Hariharan Signed-off-by: Yu Zhang Reviewed-by: Jacob Pan Reviewed-by: Michael Kelley Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/hyperv/hv-iommu-guest.c | 74 ++++++++++++++++++++++++++- include/hyperv/hvgdk_mini.h | 1 + include/hyperv/hvhdk_mini.h | 18 +++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/hyperv/hv-iommu-guest.c b/drivers/iommu/hyperv/hv-iommu-guest.c index 341d70eed592..fe06bd970621 100644 --- a/drivers/iommu/hyperv/hv-iommu-guest.c +++ b/drivers/iommu/hyperv/hv-iommu-guest.c @@ -383,11 +383,82 @@ static void hv_iommu_flush_iotlb_all(struct iommu_domain *domain) hv_flush_device_domain(to_hv_iommu_domain(domain)); } +/* + * Calculate the minimal power-of-two aligned range that covers [start, end] + * (end is inclusive). Returns a single (page_number, page_mask_shift) + * descriptor that may over-flush when the range is not naturally aligned. + */ +static void +hv_iommu_calc_flush_range(unsigned long start, unsigned long end, + union hv_iommu_flush_va *va) +{ + unsigned long aligned_start; + unsigned int sz_lg2; + + sz_lg2 = max(fls_long(start ^ end), HV_HYP_PAGE_SHIFT); + + /* + * A valid IOVA range shall not span bit 63. Use the maximum mask + * so the host can safely perform a full flush. + */ + if (WARN_ON_ONCE(sz_lg2 == BITS_PER_LONG)) + aligned_start = 0; + else + aligned_start = + start & GENMASK(BITS_PER_LONG - 1, sz_lg2); + + va->page_number = aligned_start >> HV_HYP_PAGE_SHIFT; + va->page_mask_shift = sz_lg2 - HV_HYP_PAGE_SHIFT; +} + +static void hv_flush_device_domain_list(struct hv_iommu_domain *hv_domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + u64 status; + unsigned long flags; + struct hv_input_flush_device_domain_list *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + /* Clear the fixed header and the single range entry. */ + memset(input, 0, struct_size(input, iova_list, 1)); + + input->device_domain = hv_domain->device_domain; + input->flags |= HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT; + hv_iommu_calc_flush_range(iotlb_gather->start, iotlb_gather->end, + &input->iova_list[0]); + + status = hv_do_rep_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN_LIST, + 1, 0, input, NULL); + + if (WARN_ONCE(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN_LIST failed: %#llx (%s)\n", + status, hv_result_to_string(status))) { + /* Page-selective flush failed, fall back to full flush. */ + struct hv_input_flush_device_domain *flush_all = (void *)input; + + memset(flush_all, 0, sizeof(*flush_all)); + flush_all->device_domain = hv_domain->device_domain; + status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN, + flush_all, NULL); + WARN(!hv_result_success(status), + "HVCALL_FLUSH_DEVICE_DOMAIN fallback also failed: %#llx (%s)\n", + status, hv_result_to_string(status)); + } + + local_irq_restore(flags); +} + static void hv_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *iotlb_gather) { - hv_flush_device_domain(to_hv_iommu_domain(domain)); + hv_flush_device_domain_list(to_hv_iommu_domain(domain), iotlb_gather); + /* + * The hypercall also invalidates non-leaf page-walk caches, so + * detached page-table pages can now be freed. + */ iommu_put_pages_list(&iotlb_gather->freelist); } @@ -439,6 +510,7 @@ static struct iommu_domain *hv_iommu_domain_alloc_paging(struct device *dev) cfg.common.hw_max_vasz_lg2 = hv_iommu_device->max_iova_width; cfg.common.hw_max_oasz_lg2 = 52; + cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); /* * Hyper-V S1 domains use a 4-level root for IOVA widths up to * 48 bits. A 5-level root is used only for wider apertures when diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index 5d1338197eee..966945f0504f 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h @@ -497,6 +497,7 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */ #define HVCALL_GET_GPA_PAGES_ACCESS_STATES 0x00c9 #define HVCALL_CONFIGURE_DEVICE_DOMAIN 0x00ce #define HVCALL_FLUSH_DEVICE_DOMAIN 0x00d0 +#define HVCALL_FLUSH_DEVICE_DOMAIN_LIST 0x00d1 #define HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d7 #define HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d8 #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h index 7b9f43b320cd..57b70efe60d4 100644 --- a/include/hyperv/hvhdk_mini.h +++ b/include/hyperv/hvhdk_mini.h @@ -685,4 +685,22 @@ struct hv_input_flush_device_domain { u32 reserved; } __packed; +union hv_iommu_flush_va { /* HV_IOMMU_GVA_RANGE */ + u64 as_uint64; + struct { + u64 page_mask_shift : 6; + u64 reserved : 6; + u64 page_number : 52; + }; +} __packed; + +/* HV_INPUT_FLUSH_DEVICE_DOMAIN_LIST */ +struct hv_input_flush_device_domain_list { + struct hv_input_device_domain device_domain; +#define HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT BIT(0) + u32 flags; + u32 reserved; + union hv_iommu_flush_va iova_list[]; +} __packed; + #endif /* _HV_HVHDK_MINI_H */ From 1c8f6308e557f32412e9d4678b3f029f47f6d9aa Mon Sep 17 00:00:00 2001 From: Krzysztof Karas Date: Fri, 2 Oct 2026 05:33:20 +0000 Subject: [PATCH 63/69] iommu/dma: Catch scatterlist length overflows It is possible, when a very large mapping uses only one scatterlist, that padding overflows scatterlist's length field. This results in: 1) silently wrapping the value 2) smaller than desired mappings produced by iommu_map_sg 3) leaving mapped bytes in memory (no iommu_unmap) Address this issue by adding overflow detection for scatterlist length field. Fixes: 809eac54cdd6 ("iommu/dma: Implement scatterlist segment merging") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Krzysztof Karas Reviewed-by: Robin Murphy Reviewed-by: Sebastian Brzezinka Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 1b8012123232..c19f668a7be9 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1478,6 +1478,11 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, sg_dma_len(s) = s_length; s->offset -= s_iova_off; s_length = iova_align(iovad, s_length + s_iova_off); + if (overflows_type(s_length, s->length)) { + ret = -EOVERFLOW; + + goto out_restore_sg; + } s->length = s_length; /* @@ -1494,7 +1499,20 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, * time through here (i.e. before it has a meaningful value). */ if (pad_len && pad_len < s_length - 1) { - prev->length += pad_len; + unsigned int new_prev_len; + + /* + * For large mappings spanning multiple GBs we + * may not be able to fit all needed padding into + * sg->length. + */ + if (check_add_overflow(prev->length, pad_len, &new_prev_len)) { + ret = -EOVERFLOW; + + goto out_restore_sg; + } + + prev->length = new_prev_len; iova_len += pad_len; } From fdd81f15fc43bdf3160118195431a481d99e997a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 2 Oct 2026 17:34:58 +0100 Subject: [PATCH 64/69] iommu/dma: Clean up redundant MSI cookie check The has_msi_cookie() condition only mattered transiently as an artefact of the cookie rework being done in parallel with the sw_msi rework. Once combined, calls to iommu_dma_sw_msi() are themselves dispatched based on domain->cookie_type, so checking again is clearly redundant. Signed-off-by: Robin Murphy Reviewed-by: Nicolin Chen Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c19f668a7be9..d3225d1ed531 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -2180,12 +2180,6 @@ out_err: dev_clear_dma_iommu(dev); } -static bool has_msi_cookie(const struct iommu_domain *domain) -{ - return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA || - domain->cookie_type == IOMMU_COOKIE_DMA_MSI); -} - static size_t cookie_msi_granule(const struct iommu_domain *domain) { switch (domain->cookie_type) { @@ -2267,11 +2261,6 @@ int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc, struct device *dev = msi_desc_to_dev(desc); const struct iommu_dma_msi_page *msi_page; - if (!has_msi_cookie(domain)) { - msi_desc_set_iommu_msi_iova(desc, 0, 0); - return 0; - } - iommu_group_mutex_assert(dev); msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); if (!msi_page) From 05d62be400f3ae0f6856f6910a1281e67cbaabcf Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 13 Oct 2026 13:19:43 +0100 Subject: [PATCH 65/69] iommu/amd: Program device table when kdump reuse fails Commit 38e5f33ee359 ("iommu/amd: Reuse device table for kdump") made iommu_set_device_table() return early in every kdump kernel. That is needed when the previous kernel's device table is reused: the base register already points at it, and on SNP-enabled systems the register is locked. But if reuse isn't possible, early_enable_iommus() falls back to the freshly allocated table and calls early_enable_iommu(). Because of the early return, that table is never written to the base register. The IOMMU keeps using whatever table the register pointed at before, while the driver writes entries into the new one. Only skip the write when the previous kernel's table was reused. Fixes: 38e5f33ee359 ("iommu/amd: Reuse device table for kdump") Cc: stable@vger.kernel.org Reviewed-by: Ashish Kalra Reviewed-by: Ankit Soni Reviewed-by: Vasant Hegde Signed-off-by: Matt Fleming Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 8ec8a6fccaaf..1d7b9fe305ac 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -389,7 +389,13 @@ static void iommu_set_device_table(struct amd_iommu *iommu) BUG_ON(iommu->mmio_base == NULL); - if (is_kdump_kernel()) + /* + * A kdump kernel that reuses the previous kernel's device table must + * leave the base register alone. It already points at that table, and + * with SNP enabled the register is locked. If reuse failed, program + * the freshly allocated table like a normal boot. + */ + if (iommu->pci_seg->old_dev_tbl_cpy) return; entry = iommu_virt_to_phys(dev_table); From bae0d3760661ac77148944e88aafef698cf7b2bf Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 13 Oct 2026 13:19:44 +0100 Subject: [PATCH 66/69] iommu/amd: Don't allocate kdump device table from DMA32 Booting a kdump kernel with crashkernel=X,high crashkernel=0,low on an AMD system fails the per-segment device table allocation because there is no memory below 4G: swapper/0: page allocation failure: order:9, mode:0x40104(GFP_DMA32|__GFP_ZERO|__GFP_COMP) iommu_alloc_pages_node_sz+0x7f/0x150 alloc_pci_segment+0x4bd/0x650 iommu_go_to_state+0x213d/0x34e0 amd_iommu_prepare+0x39/0xa0 irq_remapping_prepare+0x83/0xd0 enable_IR_x2apic+0x61/0x370 On our E810 systems, the kdump kernel booted but ice MSI-X interrupts never fired and the TX queues timed out. Commit b336781b82cc ("iommu/amd: Allocate memory below 4G for dev table if translation pre-enabled") allocated device tables below 4G. At the time, the kdump kernel programmed a copy of the old table into the base register with the IOMMU still enabled. The 64-bit register is updated as two 32-bit halves, so the running IOMMU could see a half-written, invalid table pointer. Keeping both tables below 4G leaves the upper half zero throughout the update, so the pointer stays valid. Since commit 38e5f33ee359 ("iommu/amd: Reuse device table for kdump") the kdump kernel reuses the previous kernel's table without writing the base register, so its newly allocated table is discarded when reuse succeeds. If reuse fails, the previous patch programs the new table only after disabling the IOMMU. Either way, the kdump kernel's table doesn't need to be below 4G. The previous kernel still allocates its table from DMA32, so the kdump kernel's check that the old table is below 4G still works. Drop GFP_DMA32 for kdump kernels only. Cc: stable@vger.kernel.org # 6.18.x Reviewed-by: Ashish Kalra Reviewed-by: Ankit Soni Signed-off-by: Matt Fleming Reviewed-by: Vasant Hegde Signed-off-by: Matt Fleming Reviewed-by: Ashish Kalra Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 1d7b9fe305ac..d5e1acf5d394 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -628,8 +628,19 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_ /* Allocate per PCI segment device table */ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, - pci_seg->dev_table_size); + gfp_t gfp = GFP_KERNEL; + + /* + * Keep normal kernel device tables below 4G because a later kdump + * kernel only trusts and reuses old tables below that limit. A kdump + * kernel's new table is either discarded when reuse succeeds or + * programmed after the IOMMU is disabled, so the table does not + * need DMA32 memory (which the crashkernel may not have). + */ + if (!is_kdump_kernel()) + gfp |= GFP_DMA32; + + pci_seg->dev_table = iommu_alloc_pages_sz(gfp, pci_seg->dev_table_size); if (!pci_seg->dev_table) return -ENOMEM; From 36f115d23fb1bdb2f0a4c0d8e31727af42b312d4 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Thu, 15 Oct 2026 16:23:50 +0800 Subject: [PATCH 67/69] iommu/hyperv: Add missing PCI dependency for the guest IOMMU driver Make HYPERV_PVIOMMU depend on PCI, to fix the possible link failure of pci_request_acs(), if HYPERV_PVIOMMU is enabled yet with PCI disabled. Such configuration shall be prevented. Fixes: 2690d4156b51 ("iommu/hyperv: Add para-virtualized IOMMU support for Hyper-V guest") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610110511.Nk1sZ8SG-lkp@intel.com/ Signed-off-by: Yu Zhang Reviewed-by: Jacob Pan Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/hyperv/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/hyperv/Kconfig b/drivers/iommu/hyperv/Kconfig index fe24ed65b6a7..f2f993f647c3 100644 --- a/drivers/iommu/hyperv/Kconfig +++ b/drivers/iommu/hyperv/Kconfig @@ -2,7 +2,7 @@ # Hyper-V para-virtualized IOMMU driver for Linux guests config HYPERV_PVIOMMU bool "Microsoft Hyper-V para-virtualized IOMMU driver for Linux guests" - depends on X86_64 && HYPERV + depends on X86_64 && HYPERV && PCI select IOMMU_DMA select GENERIC_PT select IOMMU_PT From 1e7c4390c023841e5232922117a4347275bc8c73 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Thu, 15 Oct 2026 16:23:51 +0800 Subject: [PATCH 68/69] iommu/hyperv: Make hv_iommu_device static Make hv_iommu_device static to give it internal linkage. This pointer is only used within hv-iommu-guest.c, but previously has external linkage without a prior declaration. This causes sparse to warn that the symbol was not declared. Fixes: 2690d4156b51 ("iommu/hyperv: Add para-virtualized IOMMU support for Hyper-V guest") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610092126.TuSXnTYR-lkp@intel.com/ Signed-off-by: Yu Zhang Reviewed-by: Jacob Pan Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/hyperv/hv-iommu-guest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/hyperv/hv-iommu-guest.c b/drivers/iommu/hyperv/hv-iommu-guest.c index fe06bd970621..876a22f21f1e 100644 --- a/drivers/iommu/hyperv/hv-iommu-guest.c +++ b/drivers/iommu/hyperv/hv-iommu-guest.c @@ -22,7 +22,7 @@ #include "iommu.h" #include "../iommu-pages.h" -struct hv_iommu_dev *hv_iommu_device; +static struct hv_iommu_dev *hv_iommu_device; /* * Identity and blocking domains are static singletons: identity is a 1:1 From cc2ca8750fe398fa1958a1566e5582c69f5f62bc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 15 Oct 2026 18:01:19 +0200 Subject: [PATCH 69/69] MAINTAINERS: Make Vasant Hegde AMD IOMMU maintainer Vasant reviews and tests most of the AMD IOMMU patches and is critical for maintaining the driver. Make him an official maintainer. Cc: Vasant Hegde Cc: Suravee Suthikulpanit Cc: Will Deacon Acked-by: Vasant Hegde Signed-off-by: Joerg Roedel --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cc3cae2e378b..64052e51c3cc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1181,8 +1181,8 @@ F: drivers/platform/x86/amd/hfi/ AMD IOMMU (AMD-VI) M: Joerg Roedel (AMD) +M: Vasant Hegde R: Suravee Suthikulpanit -R: Vasant Hegde L: iommu@lists.linux.dev S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git