mirrors/linux
0
0 Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-10-26 10:57:38 +08:00
Code Activity

Merge tag 'iommu-updates-v7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux

Browse Source 
Pull IOMMU updates from Joerg Roedel:
 "Core:
   - Detect scatterlist length overflows and finalize deferred domain
     attachments before mapping MSI pages
   - Consolidate x86 MSI reserved-region handling and clean up internal
     DMA and SVA helpers

  AMD IOMMU:
   - Add IOMMU PerfOpt support, enabled by amdgpu for supported
     integrated GPUs in identity domains
   - Restrict forced identity mapping to GPUs that require it
   - Fix kdump device-table allocation and reuse, ATS configuration
     error handling, and an interrupt-remapping allocation leak
   - Clean up device probing and initialization
   - Add Vasant Hegde as an AMD IOMMU maintainer

  ARM SMMUv2:
   - Device-tree binding updates for Qualcomm Maili and Nord SoCs

  ARM SMMUv3:
   - Work around hardware integration issue on Google Tensor G5 SoCs
     where all DMA transactions appear as instruction fetches
   - Allow command queue size to be configured on the kernel
     command-line
   - Reduce queue sizes when running as a kdump kernel
   - Fix some pre-existing issues in the driver and kunit tests reported
     by Sashiko
   - Fix handling of Arm hardware errata relating to the contiguous hint
     when using SVA
   - Rework the TLB invalidation logic in preparation for enabling
     iommu-pt in future
   - Minor cleanups in preparation for improving support for kdump

  Apple DART:
   - Support device-tree DMA aperture specifications

  Broadcom:
   - Add BCM2712 IOMMU support for Raspberry Pi 5 display and multimedia
     devices, including the generic page-table format and shared TLB
     cache management

  Hyper-V:
   - Add a para-virtualized guest IOMMU driver with guest-managed
     stage-1 page tables and page-selective IOTLB invalidation

  Intel VT-d:
   - Enhanced IQE fault handling for batched multiple descriptors,
   - Optimization of page-table level calculation in
     compute_vasz_lg2_ss(),
   - Fixes for several pre-existing issues reported by Sashiko,
   - General cleanup and refinement

  RISC-V:
   - Report cache-coherency capability for coherent devices, enabling
     VFIO/iommufd use

  Samsung Exynos:
   - Support SysMMUs without BLOCK mode, fixing their enable sequence
     and TLB invalidation
   - Improve v7 fault diagnostics with transaction information"

* tag 'iommu-updates-v7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux: (69 commits)
  MAINTAINERS: Make Vasant Hegde AMD IOMMU maintainer
  iommu/hyperv: Make hv_iommu_device static
  iommu/hyperv: Add missing PCI dependency for the guest IOMMU driver
  iommu/amd: Don't allocate kdump device table from DMA32
  iommu/amd: Program device table when kdump reuse fails
  iommu/dma: Clean up redundant MSI cookie check
  iommu/dma: Catch scatterlist length overflows
  iommu/hyperv: Add page-selective IOTLB flush support
  iommu/hyperv: Add para-virtualized IOMMU support for Hyper-V guest
  iommu/x86: Add architectural MSI reserved region helper
  Drivers: hv: Add logical device ID registry for vPCI devices
  hyperv: Introduce new hypercall interfaces used by Hyper-V guest IOMMU
  iommu: Make iommu-dma helpers private
  iommu/arm-smmu-v3: Don't rb_erase() a never-inserted stream node
  iommu/arm-smmu-v3: Add Q_POS() macro
  iommu/arm-smmu-v3: Add arm_smmu_attach_release()
  iommu/arm-smmu-v3: Disable the impl before disabling the SMMU on shutdown
  iommu/arm-smmu-v3: Populate the tlbi at the top of the call chain
  iommu/arm-smmu-v3: Precompute the invalidation commands
  iommu/arm-smmu-v3: Keep track in arm_smmu_invs if range invalidation is used
  ...
This commit is contained in:
Linus Torvalds committed 2026-10-23 14:48:26 -10:00
2 parents 906b6e1d8c 20bfd25757
commit 4da9dedf3f
61 files changed
+3716 -489
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/arch/arm64/silicon-errata.rst
+2 -1
View File
@@ -276,7 +276,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU L1 | #3878312 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU S3 | #3995052 | N/A |
| ARM | MMU S3 | #3995052, | N/A |
| | | #3673557 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 |
+----------------+-----------------+-----------------+-----------------------------+
Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml
+3
View File
@@ -35,6 +35,9 @@ properties:
minItems: 1
maxItems: 2
iommus:
maxItems: 1
required:
- compatible
- reg
Documentation/devicetree/bindings/iommu/apple,dart.yaml
+7 -5
View File
@@ -44,10 +44,12 @@ properties:
Optional since not all IOMMUs are attached to a clock gate.
'#iommu-cells':
const: 1
enum: [ 1, 5 ]
description:
Has to be one. The single cell describes the stream id emitted by
a master to the IOMMU.
Has to be one or five. The first cell describes the stream id emitted by
a master to the IOMMU. The optional second and third cell describe the
aperture start of a master's DMA window and fourth and fifth cell describe
the size of DMA window.
power-domains:
maxItems: 1
@@ -66,11 +68,11 @@ examples:
compatible = "apple,t8103-dart";
reg = <0x82f80000 0x4000>;
interrupts = <1 781 4>;
#iommu-cells = <1>;
#iommu-cells = <5>;
};
master1 {
iommus = <&dart1 0>;
iommus = <&dart1 0 0x100 0x0 0x10 0x0>;
};
- |+
Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
+10
View File
@@ -83,6 +83,16 @@ properties:
register access with page 0 offsets. Set for Cavium ThunderX2 silicon that
doesn't support SMMU page1 register space.
arm,instdata-override:
type: boolean
description:
Some SoCs like Google Tensor G5 (Laguna) have an SMMU integration
silicon erratum where AxPROT[2] signal is always 1, making all
transactions appear as instructions.
This property enables a software workaround to override the
instruction/data attribute of incoming traffic to data.
allOf:
- if:
not:
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
+4
View File
@@ -101,7 +101,9 @@ properties:
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
- qcom,maili-smmu-500
- qcom,milos-smmu-500
- qcom,nord-smmu-500
- qcom,qcm2290-smmu-500
- qcom,qcs615-smmu-500
- qcom,qcs8300-smmu-500
@@ -579,6 +581,8 @@ allOf:
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
- qcom,maili-smmu-500
- qcom,nord-smmu-500
- qcom,sm8750-smmu-500
- const: qcom,adreno-smmu
- const: qcom,smmu-500
Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommu.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Broadcom BCM2712 IOMMU
maintainers:
- Daniel Drake <dan@reactivated.net>
- Florian Fainelli <florian.fainelli@broadcom.com>
- Jim Quinlan <james.quinlan@broadcom.com>
description:
The BCM2712 SoC features multiple independent IOMMU instances providing
address translation for multimedia hardware blocks, such as the VC6 display
pipeline, camera receivers, and ISP. Each IOMMU has its own page tables,
control registers, and local TLB, and is additionally linked to a centralized
L2 TLB (IOMMUC).
properties:
compatible:
const: brcm,bcm2712-iommu
reg:
maxItems: 1
'#iommu-cells':
const: 0
brcm,iommu-cache:
$ref: /schemas/types.yaml#/definitions/phandle
description:
Phandle to the shared IOMMU cache (IOMMUC). Each IOMMU instance caches
translated descriptors in this centralized L2 TLB cache block and must
trigger cache invalidation commands on it when updating or unmapping page
table entries.
required:
- compatible
- reg
- '#iommu-cells'
- brcm,iommu-cache
additionalProperties: false
examples:
- |
iommu@5200 {
compatible = "brcm,bcm2712-iommu";
reg = <0x5200 0x80>;
brcm,iommu-cache = <&iommuc>;
#iommu-cells = <0>;
};
Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml
+40
View File
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommuc.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Broadcom BCM2712 Shared IOMMU Cache (IOMMUC)
maintainers:
- Daniel Drake <dan@reactivated.net>
- Florian Fainelli <florian.fainelli@broadcom.com>
- Jim Quinlan <james.quinlan@broadcom.com>
description:
The BCM2712 IOMMUC is a centralized Translation Lookaside Buffer (TLB) cache
which accelerates address translation across the SoC's IOMMU devices. If an
address mapping is not found in the IOMMU's local TLB cache, then this
IOMMUC is consulted as a L2 TLB. The SoC includes a single IOMMUC which is
shared between all IOMMUs, and it must be explicitly invalidated when
modifying or unmapping IOMMU page tables.
properties:
compatible:
const: brcm,bcm2712-iommuc
reg:
maxItems: 1
required:
- compatible
- reg
additionalProperties: false
examples:
- |
iommuc@5b00 {
compatible = "brcm,bcm2712-iommuc";
reg = <0x5b00 0x80>;
};
MAINTAINERS
+13 -1
View File
@@ -1191,8 +1191,8 @@ F: drivers/platform/x86/amd/hfi/
AMD IOMMU (AMD-VI)
M: Joerg Roedel (AMD) <joro@8bytes.org>
M: Vasant Hegde <vasant.hegde@amd.com>
R: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
R: Vasant Hegde <vasant.hegde@amd.com>
L: iommu@lists.linux.dev
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
@@ -5188,6 +5188,18 @@ N: bcm2712
N: bcm283*
N: raspberrypi
BROADCOM BCM2712 IOMMU DRIVER
M: Daniel Drake <dan@reactivated.net>
M: Florian Fainelli <florian.fainelli@broadcom.com>
M: Jim Quinlan <james.quinlan@broadcom.com>
R: Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
L: iommu@lists.linux.dev
L: linux-rpi-kernel@lists.infradead.org (moderated for non-subscribers)
S: Maintained
F: Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu*
F: drivers/iommu/bcm2712-iommu*
F: drivers/iommu/generic_pt/fmt/*bcm2712*
BROADCOM BCM281XX/BCM11XXX/BCM216XX ARM ARCHITECTURE
M: Florian Fainelli <florian.fainelli@broadcom.com>
M: Ray Jui <rjui@broadcom.com>
arch/x86/hyperv/hv_init.c
+4
View File
@@ -579,6 +579,10 @@ skip_hypercall_pg_init:
old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev;
x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev;
#ifdef CONFIG_HYPERV_PVIOMMU
x86_init.iommu.iommu_init = hv_iommu_init;
#endif
hv_apic_init();
x86_init.pci.arch_init = hv_pci_init;
arch/x86/include/asm/iommu.h
+20
View File
@@ -4,6 +4,11 @@
#include <linux/acpi.h>
#ifdef CONFIG_IOMMU_API
#include <linux/iommu.h>
#include <linux/sizes.h>
#endif
#include <asm/e820/api.h>
extern int force_iommu, no_iommu;
@@ -21,6 +26,21 @@ extern bool x86_swiotlb_enable;
/* 10 seconds */
#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
#ifdef CONFIG_IOMMU_API
static inline int iommu_add_resv_x86_msi_region(struct list_head *head)
{
struct iommu_resv_region *region;
region = iommu_alloc_resv_region(0xfee00000, SZ_1M, 0,
IOMMU_RESV_MSI, GFP_KERNEL);
if (!region)
return -ENOMEM;
list_add_tail(&region->list, head);
return 0;
}
#endif
static inline int __init
arch_rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
{
arch/x86/include/asm/mshyperv.h
+4
View File
@@ -313,6 +313,10 @@ static inline void mshv_vtl_return_hypercall(void) {}
static inline void __mshv_vtl_return_call(struct mshv_vtl_cpu_context *vtl0) {}
#endif
#ifdef CONFIG_HYPERV_PVIOMMU
int __init hv_iommu_init(void);
#endif
#include <asm-generic/mshyperv.h>
#endif
drivers/gpu/drm/amd/amdgpu/amdgpu.h
+1
View File
@@ -156,6 +156,7 @@ struct amdgpu_watchdog_timer {
* Modules parameters.
*/
extern int amdgpu_modeset;
extern int amdgpu_iommu_perfopt;
extern unsigned int amdgpu_vram_limit;
extern int amdgpu_vis_vram_limit;
extern int amdgpu_gart_size;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+50
View File
@@ -33,6 +33,7 @@
#include <linux/console.h>
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/amd-iommu.h>
#include <linux/pci.h>
#include <linux/pci-p2pdma.h>
#include <linux/apple-gmux.h>
@@ -3751,6 +3752,28 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
apple_gmux_detect(NULL, NULL)));
}
static inline bool amdgpu_device_identity(struct amdgpu_device *adev)
{
struct pci_dev *pdev = adev->pdev;
struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
if (!domain)
return false;
return domain->type == IOMMU_DOMAIN_IDENTITY;
}
static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev)
{
if (amdgpu_iommu_perfopt == 0)
return false;
if (!(adev->flags & AMD_IS_APU))
return false;
return amdgpu_device_identity(adev);
}
/**
* amdgpu_device_init - initialize the driver
*
@@ -3958,6 +3981,16 @@ int amdgpu_device_init(struct amdgpu_device *adev,
if (r)
return r;
if (amdgpu_device_use_perfopt(adev)) {
int perfopt_ret = amd_iommu_enable_perfopt(pdev);
/* Optional optimization; a failure to arm it must not abort probe. */
if (perfopt_ret)
dev_warn(adev->dev,
"Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
perfopt_ret);
}
/*
* No need to remove conflicting FBs for non-display class devices.
* This prevents the sysfb from being freed accidently.
@@ -4327,6 +4360,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
amdgpu_gart_dummy_page_fini(adev);
if (amdgpu_device_use_perfopt(adev))
amd_iommu_disable_perfopt(adev->pdev);
if (pci_dev_is_disconnected(adev->pdev))
amdgpu_device_unmap_mmio(adev);
@@ -4690,6 +4726,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
return 0;
if (amdgpu_device_use_perfopt(adev)) {
int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev);
/*
* Must not return on failure: a bare return would leak the
* SR-IOV VF exclusive-mode acquisition taken above (released
* via the exit: path).
*/
if (perfopt_ret)
dev_warn(adev->dev,
"Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
perfopt_ret);
}
if (adev->in_s0ix)
amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+12
View File
@@ -185,6 +185,7 @@ char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
int amdgpu_enforce_isolation = -1;
int amdgpu_modeset = -1;
int amdgpu_iommu_perfopt = -1;
/* Specifies the default granularity for SVM, used in buffer
* migration and restoration of backing memory when handling
@@ -392,6 +393,17 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444);
MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(aspm, amdgpu_aspm, int, 0444);
/**
* DOC: iommu_perfopt (int)
* Control the AMD IOMMU PerfOpt DMA-latency optimization
* (0 = disable; -1 = enable on supported devices).
* This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn)
* Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it,
* trading isolation for lower DMA latency.
*/
MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)");
module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444);
/**
* DOC: runpm (int)
* Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down
drivers/hv/hv_common.c
+115
View File
@@ -21,6 +21,7 @@
#include <linux/panic_notifier.h>
#include <linux/ptrace.h>
#include <linux/random.h>
#include <linux/rhashtable.h>
#include <linux/efi.h>
#include <linux/kdebug.h>
#include <linux/kmsg_dump.h>
@@ -78,6 +79,22 @@ static struct ctl_table_header *hv_ctl_table_hdr;
u8 * __percpu *hv_synic_eventring_tail;
EXPORT_SYMBOL_GPL(hv_synic_eventring_tail);
struct hv_pci_busdata {
int pci_domain_nr;
u32 logical_dev_id_prefix;
struct rhash_head node;
struct rcu_head rcu;
};
static struct rhashtable hv_pci_bus_ht;
static bool hv_pci_bus_ht_initialized;
static const struct rhashtable_params hv_pci_bus_ht_params = {
.key_len = sizeof_field(struct hv_pci_busdata, pci_domain_nr),
.key_offset = offsetof(struct hv_pci_busdata, pci_domain_nr),
.head_offset = offsetof(struct hv_pci_busdata, node),
};
/*
* Hyper-V specific initialization and shutdown code that is
* common across all architectures. Called from architecture
@@ -86,6 +103,11 @@ EXPORT_SYMBOL_GPL(hv_synic_eventring_tail);
void __init hv_common_free(void)
{
if (hv_pci_bus_ht_initialized) {
rhashtable_destroy(&hv_pci_bus_ht);
hv_pci_bus_ht_initialized = false;
}
unregister_sysctl_table(hv_ctl_table_hdr);
hv_ctl_table_hdr = NULL;
@@ -315,6 +337,7 @@ u8 __init get_vtl(void)
int __init hv_common_init(void)
{
int i;
int ret;
union hv_hypervisor_version_info version;
/* Get information about the Microsoft Hypervisor version */
@@ -394,6 +417,13 @@ int __init hv_common_init(void)
for (i = 0; i < nr_cpu_ids; i++)
hv_vp_index[i] = VP_INVAL;
ret = rhashtable_init(&hv_pci_bus_ht, &hv_pci_bus_ht_params);
if (ret) {
hv_common_free();
return ret;
}
hv_pci_bus_ht_initialized = true;
return 0;
}
@@ -808,6 +838,7 @@ static const struct hv_status_info hv_status_infos[] = {
_STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EOPNOTSUPP),
_STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL),
_STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL),
_STATUS_INFO(HV_STATUS_DEVICE_ALREADY_IN_DOMAIN, -EEXIST),
_STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO),
_STATUS_INFO(HV_STATUS_TIME_OUT, -EIO),
_STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO),
@@ -861,3 +892,87 @@ const char *hv_result_to_string(u64 status)
return "Unknown";
}
EXPORT_SYMBOL_GPL(hv_result_to_string);
/*
* Logical device ID registry for Hyper-V PCI buses. The pci-hyperv
* driver registers each bus's logical device ID prefix before scanning
* its devices. Consumers look up the prefix by PCI domain number when
* building logical device IDs for Hyper-V interfaces.
*/
int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix)
{
struct hv_pci_busdata *bus, *new;
int ret;
new = kzalloc_obj(*new, GFP_KERNEL);
if (!new)
return -ENOMEM;
new->pci_domain_nr = pci_domain_nr;
new->logical_dev_id_prefix = logical_dev_id_prefix;
bus = rhashtable_lookup_get_insert_fast(&hv_pci_bus_ht, &new->node,
hv_pci_bus_ht_params);
if (IS_ERR(bus)) {
ret = PTR_ERR(bus);
goto free_new;
}
if (WARN_ONCE(bus != NULL,
"Hyper-V PCI domain %04x is already registered\n",
pci_domain_nr)) {
ret = -EEXIST;
goto free_new;
}
return 0;
free_new:
kfree(new);
return ret;
}
EXPORT_SYMBOL_FOR_MODULES(hv_pci_register_dev_id, "pci-hyperv");
void hv_pci_unregister_dev_id(int pci_domain_nr)
{
struct hv_pci_busdata *bus;
int ret = -ENOENT;
rcu_read_lock();
bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr,
hv_pci_bus_ht_params);
if (bus)
ret = rhashtable_remove_fast(&hv_pci_bus_ht, &bus->node,
hv_pci_bus_ht_params);
rcu_read_unlock();
if (WARN_ONCE(ret,
"Failed to unregister Hyper-V PCI domain %04x\n",
pci_domain_nr))
return;
kfree_rcu(bus, rcu);
}
EXPORT_SYMBOL_FOR_MODULES(hv_pci_unregister_dev_id, "pci-hyperv");
/*
* Look up the logical device ID prefix registered for @pci_domain_nr.
* Returns 0 on success with *prefix filled in; -ENODEV if no entry is
* registered for that PCI domain.
*/
int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix)
{
struct hv_pci_busdata *bus;
int ret = -ENODEV;
rcu_read_lock();
bus = rhashtable_lookup(&hv_pci_bus_ht, &pci_domain_nr,
hv_pci_bus_ht_params);
if (bus) {
*prefix = bus->logical_dev_id_prefix;
ret = 0;
}
rcu_read_unlock();
return ret;
}
drivers/iommu/Kconfig
+17 -1
View File
@@ -195,6 +195,7 @@ config MSM_IOMMU
source "drivers/iommu/amd/Kconfig"
source "drivers/iommu/arm/Kconfig"
source "drivers/iommu/intel/Kconfig"
source "drivers/iommu/hyperv/Kconfig"
source "drivers/iommu/iommufd/Kconfig"
source "drivers/iommu/riscv/Kconfig"
@@ -303,7 +304,7 @@ config SPAPR_TCE_IOMMU
config APPLE_DART
tristate "Apple DART IOMMU Support"
depends on ARCH_APPLE || COMPILE_TEST
depends on ARCH_APPLE || (COMPILE_TEST && 64BIT)
depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART
select IOMMU_API
select IOMMU_IO_PGTABLE_DART
@@ -386,6 +387,21 @@ config VSI_IOMMU
Say Y here if you want to use this IOMMU in front of these
hardware blocks.
config BCM2712_IOMMU
bool "BCM2712 IOMMU driver"
depends on (ARCH_BCM2835 && ARM64) || COMPILE_TEST
select IOMMU_API
select GENERIC_PT
select IOMMU_PT
select IOMMU_PT_BCM2712
help
Support for IOMMU on BCM2712 SoC. This IOMMU can be used by the
display controller and various multimedia devices to perform
efficient memory management.
Say Y here if you want to use this IOMMU in front of these
hardware blocks.
config IOMMU_DEBUG_PAGEALLOC
bool "Debug IOMMU mappings against page allocations"
depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
drivers/iommu/Makefile
+1
View File
@@ -37,4 +37,5 @@ obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
obj-$(CONFIG_APPLE_DART) += apple-dart.o
obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o
obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o
obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o
drivers/iommu/amd/amd_iommu.h
+3
View File
@@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize;
extern unsigned long amd_iommu_pgsize_bitmap;
extern bool amd_iommu_hatdis;
int amd_iommu_perfopt_clear(struct amd_iommu *iommu);
int amd_iommu_perfopt_restore(struct amd_iommu *iommu);
/* Protection domain ops */
void amd_iommu_init_identity_domain(void);
struct protection_domain *protection_domain_alloc(void);
drivers/iommu/amd/amd_iommu_types.h
+7
View File
@@ -53,6 +53,7 @@
#define MMIO_MSI_ADDR_LO_OFFSET 0x015C
#define MMIO_MSI_ADDR_HI_OFFSET 0x0160
#define MMIO_MSI_DATA_OFFSET 0x0164
#define MMIO_PERF_OPT_OFFSET 0x016C
#define MMIO_INTCAPXT_EVT_OFFSET 0x0170
#define MMIO_INTCAPXT_PPR_OFFSET 0x0178
#define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
@@ -87,6 +88,8 @@
#define FEATURE_GLX GENMASK_ULL(15, 14)
#define FEATURE_GAM_VAPIC BIT_ULL(21)
#define FEATURE_PASMAX GENMASK_ULL(36, 32)
#define FEATURE_PERF_OPT BIT_ULL(45)
#define PERF_OPT_EN BIT(13)
#define FEATURE_GIOSUP BIT_ULL(48)
#define FEATURE_HASUP BIT_ULL(49)
#define FEATURE_EPHSUP BIT_ULL(50)
@@ -657,6 +660,9 @@ struct amd_iommu {
/* Extended features 2 */
u64 features2;
/* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */
int perfopt_refcount;
/* PCI device id of the IOMMU device */
u16 devid;
@@ -818,6 +824,7 @@ struct iommu_dev_data {
u8 ppr :1; /* Enable device PPR support */
bool use_vapic; /* Enable device to use vapic mode */
bool defer_attach;
bool perfopt;
struct ratelimit_state rs; /* Ratelimit IOPF messages */
};
drivers/iommu/amd/init.c
+131 -61
View File
@@ -163,7 +163,6 @@ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
static bool amd_iommu_detected;
static bool amd_iommu_disabled __initdata;
static bool amd_iommu_force_enable __initdata;
static bool amd_iommu_irtcachedis;
@@ -390,7 +389,13 @@ static void iommu_set_device_table(struct amd_iommu *iommu)
BUG_ON(iommu->mmio_base == NULL);
if (is_kdump_kernel())
/*
* A kdump kernel that reuses the previous kernel's device table must
* leave the base register alone. It already points at that table, and
* with SNP enabled the register is locked. If reuse failed, program
* the freshly allocated table like a normal boot.
*/
if (iommu->pci_seg->old_dev_tbl_cpy)
return;
entry = iommu_virt_to_phys(dev_table);
@@ -623,8 +628,19 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_
/* Allocate per PCI segment device table */
static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
{
pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
pci_seg->dev_table_size);
gfp_t gfp = GFP_KERNEL;
/*
* Keep normal kernel device tables below 4G because a later kdump
* kernel only trusts and reuses old tables below that limit. A kdump
* kernel's new table is either discarded when reuse succeeds or
* programmed after the IOMMU is disabled, so the table does not
* need DMA32 memory (which the crashkernel may not have).
*/
if (!is_kdump_kernel())
gfp |= GFP_DMA32;
pci_seg->dev_table = iommu_alloc_pages_sz(gfp, pci_seg->dev_table_size);
if (!pci_seg->dev_table)
return -ENOMEM;
@@ -916,13 +932,13 @@ static void free_ga_log(struct amd_iommu *iommu)
}
#ifdef CONFIG_IRQ_REMAP
static int iommu_ga_log_enable(struct amd_iommu *iommu)
static bool iommu_ga_log_enable(struct amd_iommu *iommu)
{
u32 status, i;
u64 entry;
if (!iommu->ga_log)
return -EINVAL;
return false;
entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
@@ -946,35 +962,31 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
}
if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
return -EINVAL;
return false;
return 0;
return true;
}
#endif /* CONFIG_IRQ_REMAP */
static int iommu_init_ga_log(struct amd_iommu *iommu)
static int alloc_ga_log(struct amd_iommu *iommu)
{
#ifdef CONFIG_IRQ_REMAP
int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)))
return -EINVAL;
if (iommu->ga_log && iommu->ga_log_tail)
return 0;
iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
if (!iommu->ga_log)
goto err_out;
return -ENOMEM;
iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8);
if (!iommu->ga_log_tail)
goto err_out;
if (!iommu->ga_log_tail) {
iommu_free_pages(iommu->ga_log);
iommu->ga_log = NULL;
return -ENOMEM;
}
#endif
return 0;
err_out:
free_ga_log(iommu);
return -EINVAL;
}
#endif /* CONFIG_IRQ_REMAP */
static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
{
@@ -1948,6 +1960,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
if (!iommu->mmio_base)
return -ENOMEM;
if (amd_iommu_perfopt_clear(iommu))
pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index);
return init_iommu_from_acpi(iommu, h);
}
@@ -2152,6 +2167,27 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu)
}
}
/* Must be called after SNP support check is complete (iommu_snp_enable()) */
static bool check_vapic_support(void)
{
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
return false;
if (!check_feature(FEATURE_GAM_VAPIC)) {
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return false;
}
if (amd_iommu_snp_en &&
!FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
pr_warn("Force to disable Virtual APIC due to SNP\n");
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return false;
}
return true;
}
static int __init iommu_init_pci(struct amd_iommu *iommu)
{
int cap_ptr = iommu->cap_ptr;
@@ -2196,6 +2232,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
return -ENOMEM;
if (check_vapic_support()) {
ret = alloc_ga_log(iommu);
if (ret)
return ret;
}
if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
pr_info("Using strict mode due to virtualization\n");
iommu_set_dma_strict();
@@ -3010,22 +3052,9 @@ static void enable_iommus_vapic(void)
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
return;
if (!check_feature(FEATURE_GAM_VAPIC)) {
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return;
}
if (amd_iommu_snp_en &&
!FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
pr_warn("Force to disable Virtual APIC due to SNP\n");
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return;
}
/* Enabling GAM and SNPAVIC support */
for_each_iommu(iommu) {
if (iommu_init_ga_log(iommu) ||
iommu_ga_log_enable(iommu))
if (!iommu_ga_log_enable(iommu))
return;
iommu_feature_enable(iommu, CONTROL_GAM_EN);
@@ -3038,10 +3067,46 @@ static void enable_iommus_vapic(void)
#endif
}
static int clear_perfopt_all(void)
{
struct amd_iommu *iommu;
int err, ret = 0;
for_each_iommu(iommu) {
err = amd_iommu_perfopt_clear(iommu);
if (err)
ret = err;
}
return ret;
}
static int restore_perfopt_all(void)
{
struct amd_iommu *iommu;
int err, ret = 0;
for_each_iommu(iommu) {
err = amd_iommu_perfopt_restore(iommu);
if (err)
ret = err;
}
return ret;
}
static void disable_iommus(void)
{
struct amd_iommu *iommu;
/*
* PerfOpt is an optional performance bit, so a failure to clear it must
* not skip the mandatory disable below. This also runs from the void
* amd_iommu_disable() shutdown/kexec path, which cannot report an error.
*/
if (clear_perfopt_all())
pr_err("Failed to clear PerfOpt while disabling IOMMUs\n");
for_each_iommu(iommu)
iommu_disable(iommu);
@@ -3067,6 +3132,10 @@ static void amd_iommu_resume(void *data)
for_each_iommu(iommu)
early_enable_iommu(iommu);
/* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */
if (restore_perfopt_all())
pr_err("Failed to restore PerfOpt after IOMMU resume\n");
iommu_enable_event_buffer();
amd_iommu_enable_interrupts();
}
@@ -3228,9 +3297,6 @@ static int __init early_amd_iommu_init(void)
acpi_status status;
u8 efr_hats, max_vasize;
if (!amd_iommu_detected)
return -ENODEV;
status = acpi_get_table("IVRS", 0, &ivrs_base);
if (status == AE_NOT_FOUND)
return -ENODEV;
@@ -3309,7 +3375,7 @@ static int __init early_amd_iommu_init(void)
}
/* Disable any previously enabled IOMMUs */
if (!is_kdump_kernel() || amd_iommu_disabled)
if (!is_kdump_kernel())
disable_iommus();
if (amd_iommu_irq_remap)
@@ -3404,12 +3470,6 @@ static __init void iommu_snp_enable(void)
if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
return;
/* SNP support required IOMMU to be ON */
if (no_iommu) {
pr_warn("SNP: IOMMU disabled, SNP cannot be supported.\n");
goto disable_snp;
}
amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP);
/*
* If SNP page mode 0 is not enabled, then SNP support requires that IOMMU
@@ -3476,6 +3536,8 @@ static void amd_iommu_apply_erratum_snp(void)
#endif
}
static bool amd_iommu_sme_check(void);
/****************************************************************************
*
* AMD IOMMU Initialization State Machine
@@ -3488,7 +3550,13 @@ static int __init state_next(void)
switch (init_state) {
case IOMMU_START_STATE:
if (!detect_ivrs()) {
if (no_iommu || amd_iommu_disabled) {
init_state = IOMMU_CMDLINE_DISABLED;
ret = -EINVAL;
} else if (!amd_iommu_sme_check()) {
init_state = IOMMU_INIT_ERROR;
ret = -EINVAL;
} else if (!detect_ivrs()) {
init_state = IOMMU_NOT_FOUND;
ret = -ENODEV;
} else {
@@ -3496,13 +3564,8 @@ static int __init state_next(void)
}
break;
case IOMMU_IVRS_DETECTED:
if (amd_iommu_disabled) {
init_state = IOMMU_CMDLINE_DISABLED;
ret = -EINVAL;
} else {
ret = early_amd_iommu_init();
init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
}
ret = early_amd_iommu_init();
init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
break;
case IOMMU_ACPI_FINISHED:
early_enable_iommus();
@@ -3574,6 +3637,19 @@ static int __init iommu_go_to_state(enum iommu_init_state state)
{
int ret = -EINVAL;
/*
* Some essential housekeeping work is done by amd_iommu_detect().
* Skipping calling it implies that the platform (e.g., Xen hypervisor)
* has taken over the hardware. Progressing the state machine in this
* case is worthless and fragile.
*
* There are several paths requesting later states, so disallow implicit
* START_STATE => IVRS_DETECTED transition to prevent these paths from
* accidentally progressing the state machine.
*/
if (init_state == IOMMU_START_STATE && state != IOMMU_IVRS_DETECTED)
goto out;
while (init_state != state) {
if (init_state == IOMMU_NOT_FOUND ||
init_state == IOMMU_INIT_ERROR ||
@@ -3582,6 +3658,7 @@ static int __init iommu_go_to_state(enum iommu_init_state state)
ret = state_next();
}
out:
/*
* SNP platform initilazation requires IOMMUs to be fully configured.
* If the SNP support on IOMMUs has NOT been checked, simply mark SNP
@@ -3701,17 +3778,10 @@ void __init amd_iommu_detect(void)
{
int ret;
if (no_iommu || (iommu_detected && !gart_iommu_aperture))
goto disable_snp;
if (!amd_iommu_sme_check())
goto disable_snp;
ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
if (ret)
goto disable_snp;
amd_iommu_detected = true;
iommu_detected = 1;
x86_init.iommu.iommu_init = amd_iommu_init;
return;
drivers/iommu/amd/iommu.c
+351 -117
View File
@@ -50,8 +50,6 @@
#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
/* Reserved IOVA ranges */
#define MSI_RANGE_START (0xfee00000)
#define MSI_RANGE_END (0xfeefffff)
#define HT_RANGE_START (0xfd00000000ULL)
#define HT_RANGE_END (0xffffffffffULL)
@@ -573,10 +571,17 @@ static inline int pdev_enable_cap_ats(struct pci_dev *pdev)
if (amd_iommu_iotlb_sup &&
(dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {
ret = pci_enable_ats(pdev, PAGE_SHIFT);
if (!ret) {
dev_data->ats_enabled = 1;
dev_data->ats_qdep = pci_ats_queue_depth(pdev);
}
/*
* pci_enable_ats() should not fail here because earlier
* checks have already verified support & config.
*/
if (WARN_ON(ret))
return ret;
dev_data->ats_enabled = 1;
dev_data->ats_qdep = pci_ats_queue_depth(pdev);
ret = 0;
}
return ret;
@@ -675,7 +680,8 @@ static void pdev_disable_caps(struct pci_dev *pdev)
* This function checks if the driver got a valid device from the caller to
* avoid dereferencing invalid pointers.
*/
static bool check_device(struct device *dev)
static bool lookup_device(struct device *dev,
struct amd_iommu **iommu_out, u16 *devid_out)
{
struct amd_iommu_pci_seg *pci_seg;
struct amd_iommu *iommu;
@@ -690,7 +696,7 @@ static bool check_device(struct device *dev)
devid = PCI_SBDF_TO_DEVID(sbdf);
iommu = rlookup_amd_iommu(dev);
if (!iommu)
if (!iommu || !iommu->iommu.ops)
return false;
/* Out of our scope? */
@@ -698,65 +704,35 @@ static bool check_device(struct device *dev)
if (devid > pci_seg->last_bdf)
return false;
*iommu_out = iommu;
*devid_out = devid;
return true;
}
static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
static struct iommu_dev_data *iommu_init_device(struct amd_iommu *iommu,
struct device *dev, u16 devid)
{
struct iommu_dev_data *dev_data;
int devid, sbdf;
if (dev_iommu_priv_get(dev))
return 0;
sbdf = get_device_sbdf_id(dev);
if (sbdf < 0)
return sbdf;
devid = PCI_SBDF_TO_DEVID(sbdf);
dev_data = find_dev_data(iommu, devid);
if (!dev_data)
return -ENOMEM;
return ERR_PTR(-ENOMEM);
dev_data->dev = dev;
/*
* The dev_iommu_priv_set() needes to be called before setup_aliases.
* The dev_iommu_priv_set() needs to be called before setup_aliases.
* Otherwise, subsequent call to dev_iommu_priv_get() will fail.
*/
dev_iommu_priv_set(dev, dev_data);
setup_aliases(iommu, dev);
/*
* By default we use passthrough mode for IOMMUv2 capable device.
* But if amd_iommu=force_isolation is set (e.g. to debug DMA to
* invalid address), we ignore the capability for the device so
* it'll be forced to go into translation mode.
*/
if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
dev_data->flags = pdev_get_caps(to_pci_dev(dev));
}
/* Wait for DTE updates to go through */
iommu_completion_wait(iommu);
return 0;
return dev_data;
}
static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)
{
struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
struct dev_table_entry *dev_table = get_dev_table(iommu);
int devid, sbdf;
sbdf = get_device_sbdf_id(dev);
if (sbdf < 0)
return;
devid = PCI_SBDF_TO_DEVID(sbdf);
pci_seg->rlookup_table[devid] = NULL;
memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));
setup_aliases(iommu, dev);
}
/****************************************************************************
@@ -2392,6 +2368,9 @@ static int attach_device(struct device *dev,
if (ret)
goto out;
if (dev_data->perfopt)
goto skip_caps;
/* Setup GCR3 table */
if (pdom_is_sva_capable(domain)) {
ret = init_gcr3_table(dev_data, domain);
@@ -2416,6 +2395,7 @@ static int attach_device(struct device *dev,
pdev_enable_cap_ats(pdev);
}
skip_caps:
/* Update data structures */
dev_data->domain = domain;
spin_lock_irqsave(&domain->lock, flags);
@@ -2484,73 +2464,289 @@ out:
mutex_unlock(&dev_data->mutex);
}
static struct iommu_device *amd_iommu_probe_device(struct device *dev)
static int iommu_init_device_caps(struct iommu_dev_data *dev_data,
struct device *dev,
struct amd_iommu *iommu)
{
struct iommu_device *iommu_dev;
struct amd_iommu *iommu;
struct iommu_dev_data *dev_data;
int ret;
if (!check_device(dev))
return ERR_PTR(-ENODEV);
iommu = rlookup_amd_iommu(dev);
if (!iommu)
return ERR_PTR(-ENODEV);
/* Not registered yet? */
if (!iommu->iommu.ops)
return ERR_PTR(-ENODEV);
if (dev_iommu_priv_get(dev))
return &iommu->iommu;
ret = iommu_init_device(iommu, dev);
if (ret) {
dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
iommu_dev = ERR_PTR(ret);
iommu_ignore_device(iommu, dev);
goto out_err;
}
amd_iommu_set_pci_msi_domain(dev, iommu);
iommu_dev = &iommu->iommu;
/*
* If IOMMU and device supports PASID then it will contain max
* supported PASIDs, else it will be zero.
*/
dev_data = dev_iommu_priv_get(dev);
if (amd_iommu_pasid_supported() && dev_is_pci(dev) &&
pdev_pasid_supported(dev_data)) {
dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
pci_max_pasids(to_pci_dev(dev)));
}
if (amd_iommu_pgtable == PD_MODE_NONE) {
pr_warn_once("%s: DMA translation not supported by iommu.\n",
__func__);
iommu_dev = ERR_PTR(-ENODEV);
goto out_err;
}
iommu_completion_wait(iommu);
if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;
else
dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;
if (dev_is_pci(dev))
pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
amd_iommu_set_pci_msi_domain(dev, iommu);
if (!dev_is_pci(dev))
return 0;
/*
* By default we use passthrough mode for IOMMUv2 capable device.
* But if amd_iommu=force_isolation is set (e.g. to debug DMA to
* invalid address), we ignore the capability for the device so
* it'll be forced to go into translation mode.
*/
if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
amd_iommu_gt_ppr_supported()) {
dev_data->flags = pdev_get_caps(to_pci_dev(dev));
}
/*
* If IOMMU and device supports PASID then it will contain max
* supported PASIDs, else it will be zero.
*/
if (amd_iommu_pasid_supported() &&
pdev_pasid_supported(dev_data)) {
dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
pci_max_pasids(to_pci_dev(dev)));
}
if (pci_ats_supported(to_pci_dev(dev))) {
ret = pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
if (ret)
return ret;
}
return 0;
}
/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */
static int __perfopt_write(struct amd_iommu *iommu, bool enable)
{
u32 old, val, readback;
if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
return enable ? -ENODEV : 0;
old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (old == U32_MAX)
return -EIO;
val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
if (val != old)
writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (readback == U32_MAX ||
(readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
return -EIO;
return 0;
}
/*
* PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is
* reference counted: armed on the first requesting device, cleared on the last.
*/
static int perfopt_get(struct amd_iommu *iommu)
{
unsigned long flags;
int ret = 0;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
if (iommu->perfopt_refcount == 0) {
ret = __perfopt_write(iommu, true);
if (ret)
goto out;
}
iommu->perfopt_refcount++;
out:
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
static int perfopt_put(struct amd_iommu *iommu)
{
unsigned long flags;
int ret = 0;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0)
ret = __perfopt_write(iommu, false);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
/*
* Force PERF_OPT_EN off without touching the refcount (used on init, shutdown,
* and suspend). The count is preserved so amd_iommu_perfopt_restore() can
* re-arm on resume.
*/
int amd_iommu_perfopt_clear(struct amd_iommu *iommu)
{
unsigned long flags;
int ret;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __perfopt_write(iommu, false);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
/*
* Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on
* resume, so devices armed before suspend keep the optimization without each
* consumer driver re-arming.
*/
int amd_iommu_perfopt_restore(struct amd_iommu *iommu)
{
unsigned long flags;
int ret;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
int amd_iommu_enable_perfopt(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
struct protection_domain *domain;
int ret;
if (!iommu || !dev_data)
return -ENODEV;
if (!(iommu->features & FEATURE_PERF_OPT))
return -ENODEV;
domain = dev_data->domain;
if (!domain)
return -ENODEV;
/* Already armed for this device (e.g. re-entry on resume). */
if (dev_data->perfopt)
return 0;
/*
* The bit is only architecturally valid while the device is untranslated:
* identity domain with ATS/PRI/PASID off. The identity domain is
* SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3
* table. Re-home the device onto the same identity domain with
* perfopt set, so the attach_device() skip_caps path leaves
* ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach
* pattern used by amd_iommu_attach_device().
*
* Locking: this and amd_iommu_disable_perfopt() run only from the
* consumer driver's bind/unbind path. group->mutex is not exposed to
* drivers, but a device bound to its native driver cannot have its domain
* changed concurrently by the core (VFIO ownership is mutually exclusive;
* sysfs domain changes require an unused group), so the detach/attach pair
* is serialized without it.
*/
dev_data->perfopt = true;
detach_device(&pdev->dev);
ret = attach_device(&pdev->dev, domain);
if (ret)
goto err_restore;
ret = perfopt_get(iommu);
if (ret)
goto err_rearm;
dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index);
return 0;
err_rearm:
detach_device(&pdev->dev);
err_restore:
dev_data->perfopt = false;
if (attach_device(&pdev->dev, domain))
pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n");
dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n",
iommu->index, ret);
return ret;
}
EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt);
void amd_iommu_disable_perfopt(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
struct protection_domain *domain;
if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain)
return;
if (WARN_ON(perfopt_put(iommu)))
pci_err(pdev, "failed to clear PerfOpt\n");
/*
* Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its
* identity domain with the flag cleared, so a later bind without PerfOpt
* sees a normally-capable device. See the locking note in
* amd_iommu_enable_perfopt().
*/
domain = dev_data->domain;
dev_data->perfopt = false;
detach_device(&pdev->dev);
if (attach_device(&pdev->dev, domain))
pci_err(pdev, "failed to restore caps after PerfOpt disable\n");
}
EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt);
static struct iommu_device *amd_iommu_probe_device(struct device *dev)
{
struct iommu_device *iommu_dev;
struct amd_iommu *iommu;
struct iommu_dev_data *dev_data;
u16 devid;
int ret;
if (!lookup_device(dev, &iommu, &devid))
return ERR_PTR(-ENODEV);
if (dev_iommu_priv_get(dev))
return &iommu->iommu;
dev_data = iommu_init_device(iommu, dev, devid);
if (IS_ERR(dev_data)) {
dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
return ERR_CAST(dev_data);
}
ret = iommu_init_device_caps(dev_data, dev, iommu);
if (ret)
return ERR_PTR(ret);
iommu_dev = &iommu->iommu;
/*
* When DMA translation is unavailable return error so the iommu core
* won't attempt domain attach for this device, while preserving its
* rlookup entry for interrupt remapping.
*/
if (amd_iommu_pgtable == PD_MODE_NONE) {
pr_warn_once("%s: DMA translation not supported by iommu.\n",
__func__);
return ERR_PTR(-ENODEV);
}
out_err:
return iommu_dev;
}
static void amd_iommu_release_device(struct device *dev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
if (dev_data->perfopt) {
if (WARN_ON(perfopt_put(iommu)))
dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n",
iommu->index);
dev_data->perfopt = false;
}
WARN_ON(dev_data->domain);
@@ -2925,6 +3121,19 @@ static int blocked_domain_attach_device(struct iommu_domain *domain,
struct iommu_domain *old)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
/*
* blocked_domain is also the .release_domain, so this is the normal
* teardown path: drop the reference and clear the flag here too, and
* don't fail teardown if the WARN-guarded write doesn't stick.
*/
if (dev_data->perfopt) {
if (WARN_ON(perfopt_put(iommu)))
dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n",
iommu->index);
dev_data->perfopt = false;
}
if (dev_data->domain)
detach_device(dev);
@@ -2993,6 +3202,9 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev,
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
int ret;
if (dev_data->perfopt && !pdom_is_in_pt_mode(domain))
return -EBUSY;
/*
* Skip attach device to domain if new domain is same as
* devices current domain
@@ -3137,12 +3349,8 @@ static void amd_iommu_get_resv_regions(struct device *dev,
list_add_tail(&region->list, head);
}
region = iommu_alloc_resv_region(MSI_RANGE_START,
MSI_RANGE_END - MSI_RANGE_START + 1,
0, IOMMU_RESV_MSI, GFP_KERNEL);
if (!region)
if (iommu_add_resv_x86_msi_region(head))
return;
list_add_tail(&region->list, head);
if (amd_iommu_ht_range_ignore())
return;
@@ -3162,6 +3370,26 @@ static bool amd_iommu_is_attach_deferred(struct device *dev)
return dev_data->defer_attach;
}
static bool quirks_force_identity_mapping(struct pci_dev *pdev)
{
int class = pdev->class >> 8;
/* AMD GPU vendor ID */
if (pdev->vendor != PCI_VENDOR_ID_ATI)
return false;
/* GPU class */
if (class != PCI_CLASS_DISPLAY_VGA &&
class != PCI_CLASS_DISPLAY_OTHER)
return false;
if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI)
return false;
/* It is the GPU in an APU, force identity domain */
return true;
}
static int amd_iommu_def_domain_type(struct device *dev)
{
struct iommu_dev_data *dev_data;
@@ -3170,20 +3398,23 @@ static int amd_iommu_def_domain_type(struct device *dev)
if (!dev_data)
return 0;
if (!dev_is_pci(dev))
return 0;
/* Always use DMA domain for untrusted device */
if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
if (to_pci_dev(dev)->untrusted)
return IOMMU_DOMAIN_DMA;
/*
* Do not identity map IOMMUv2 capable devices when:
* - memory encryption is active, because some of those devices
* (AMD GPUs) don't have the encryption bit in their DMA-mask
* and require remapping.
* - SNP is enabled, because it prohibits DTE[Mode]=0.
*/
if (pdev_pasid_supported(dev_data) &&
!cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
!amd_iommu_snp_en) {
/* Apply device specific quirks */
if (quirks_force_identity_mapping(to_pci_dev(dev))) {
/*
* When memory encryption is active, some of these devices
* don't have the encryption bit in their DMA-mask and
* require remapping.
*/
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
return 0;
return IOMMU_DOMAIN_IDENTITY;
}
@@ -3902,8 +4133,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
out_free_data:
for (i--; i >= 0; i--) {
irq_data = irq_domain_get_irq_data(domain, virq + i);
if (irq_data)
kfree(irq_data->chip_data);
if (irq_data && irq_data->chip_data) {
data = irq_data->chip_data;
kfree(data->entry);
kfree(data);
}
}
for (i = 0; i < nr_irqs; i++)
free_irte(iommu, devid, index + i);
drivers/iommu/apple-dart.c
+65 -9
View File
@@ -21,6 +21,7 @@
#include <linux/io-pgtable.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/minmax.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_address.h>
@@ -267,6 +268,7 @@ struct apple_dart_domain {
struct io_pgtable_ops *pgtbl_ops;
bool finalized;
u64 mask;
struct mutex init_lock;
struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
@@ -285,6 +287,13 @@ struct apple_dart_master_cfg {
/* Intersection of DART capabilitles */
u32 supports_bypass : 1;
/*
* DMA aperture start and end to be overridden by "iommus"' phandle
* args. By default determined by DART's ias but may be outside of it.
*/
dma_addr_t dma_min;
dma_addr_t dma_max;
struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
};
@@ -537,7 +546,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
if (!ops)
return 0;
return ops->iova_to_phys(ops, iova);
return ops->iova_to_phys(ops, iova & dart_domain->mask);
}
static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
@@ -551,8 +560,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
if (!ops)
return -ENODEV;
return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
mapped);
return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize,
pgcount, prot, gfp, mapped);
}
static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
@@ -563,7 +572,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
struct apple_dart_domain *dart_domain = to_dart_domain(domain);
struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount,
gather);
}
static void
@@ -590,6 +600,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
{
struct apple_dart *dart = cfg->stream_maps[0].dart;
struct io_pgtable_cfg pgtbl_cfg;
u32 ias = min_t(u32, dart->ias, fls64(cfg->dma_max));
int ret = 0;
int i, j;
@@ -610,7 +621,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
pgtbl_cfg = (struct io_pgtable_cfg){
.pgsize_bitmap = dart->pgsize,
.ias = dart->ias,
.ias = ias,
.oas = dart->oas,
.coherent_walk = 1,
.iommu_dev = dart->dev,
@@ -623,10 +634,10 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
goto done;
}
dart_domain->mask = DMA_BIT_MASK(pgtbl_cfg.ias);
dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
dart_domain->domain.geometry.aperture_start = 0;
dart_domain->domain.geometry.aperture_end =
(dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
dart_domain->domain.geometry.aperture_start = cfg->dma_min;
dart_domain->domain.geometry.aperture_end = cfg->dma_max;
dart_domain->domain.geometry.force_aperture = true;
dart_domain->finalized = true;
@@ -803,20 +814,65 @@ static int apple_dart_of_xlate(struct device *dev,
struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
struct apple_dart *cfg_dart;
dma_addr_t dma_max = DMA_BIT_MASK(dart->ias);
dma_addr_t dma_min = 0;
int i, sid;
put_device(&iommu_pdev->dev);
if (args->args_count != 1)
if (args->args_count != 1 && args->args_count != 5)
return -EINVAL;
sid = args->args[0];
if (args->args_count == 5) {
dma_addr_t length = ((dma_addr_t)args->args[3] << 32) | args->args[4];
if (!length)
return -EINVAL;
dma_min = ((dma_addr_t)args->args[1] << 32) | args->args[2];
if (!IS_ALIGNED(dma_min, dart->pgsize) ||
!IS_ALIGNED(length, dart->pgsize)) {
dev_err(dev, "Unaligned DMA window %pad, %pad (0x%x)\n",
&dma_min, &length, dart->pgsize);
return -EINVAL;
}
if (check_add_overflow(dma_min, length - 1, &dma_max)) {
dev_err(dev, "DMA window length (%pad) overflows range for start %pad\n",
&length, &dma_min);
return -EINVAL;
}
/*
* Ensure that the DMA window does not exceed the DART's ias.
*/
if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(dart->ias)) {
dev_err(dev, "Invalid DMA window for ias=%d\n",
dart->ias);
return -EINVAL;
}
}
if (!cfg) {
cfg = kzalloc_obj(*cfg);
if (!cfg)
return -ENOMEM;
/* Will be ANDed with DART capabilities */
cfg->supports_bypass = true;
/* Will be merged with other DARTs to the common range. */
cfg->dma_min = dma_min;
cfg->dma_max = dma_max;
} else {
if (dma_min >= cfg->dma_max || cfg->dma_min >= dma_max) {
dev_err(dev, "non-overlapping DMA windows: %pad..%pad, %pad..%pad\n",
&dma_min, &dma_max,
&cfg->dma_min, &cfg->dma_max);
return -EINVAL;
}
cfg->dma_min = max(dma_min, cfg->dma_min);
cfg->dma_max = min(dma_max, cfg->dma_max);
}
dev_iommu_priv_set(dev, cfg);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+1
View File
@@ -194,6 +194,7 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain,
arm_smmu_install_ste_for_dev(master, &ste);
arm_smmu_attach_commit(&state);
mutex_unlock(&arm_smmu_asid_lock);
arm_smmu_attach_release(&state);
return 0;
}
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+20 -9
View File
@@ -140,16 +140,19 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
{
struct arm_smmu_domain *smmu_domain =
container_of(mn, struct arm_smmu_domain, mmu_notifier);
size_t size;
struct arm_smmu_tlbi tlbi = {
.tgsz_lg2 = smmu_domain->tgsz_lg2,
.iova = start,
/*
* The mm_types defines vm_end as the first byte after the end
* address, different from IOMMU subsystem using the last
* address of an address range.
*/
.size = end - start,
.iopte_size = PAGE_SIZE,
};
/*
* The mm_types defines vm_end as the first byte after the end address,
* different from IOMMU subsystem using the last address of an address
* range. So do a simple translation here by calculating size correctly.
*/
size = end - start;
arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false);
arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -215,6 +218,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
if (system_supports_haft())
feat_mask |= ARM_SMMU_FEAT_HAFT;
/*
* The workaround for ARM_SMMU_OPT_FULL_CONT_RANGE_INV requires range
* invalidation support.
*/
if (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)
feat_mask |= ARM_SMMU_FEAT_RANGE_INV;
if ((smmu->features & feat_mask) != feat_mask)
return false;
@@ -338,6 +348,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
* ARM_SMMU_FEAT_RANGE_INV is present
*/
smmu_domain->domain.pgsize_bitmap = PAGE_SIZE;
smmu_domain->tgsz_lg2 = PAGE_SHIFT;
smmu_domain->stage = ARM_SMMU_DOMAIN_SVA;
smmu_domain->smmu = smmu;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
+38 -19
View File
@@ -24,7 +24,9 @@ struct arm_smmu_test_writer {
static struct arm_smmu_ste bypass_ste;
static struct arm_smmu_ste abort_ste;
static struct arm_smmu_device smmu = {
.features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR
.features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR |
ARM_SMMU_FEAT_RANGE_INV,
.options = ARM_SMMU_OPT_FULL_CONT_RANGE_INV,
};
static struct mm_struct sva_mm = {
.pgd = (void *)0xdaedbeefdeadbeefULL,
@@ -125,8 +127,8 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer,
{
__le64 used_bits[NUM_ENTRY_QWORDS] = {};
arm_smmu_get_ste_used(ste, used_bits);
pr_debug("STE used bits: ");
writer->ops->get_used(ste, used_bits);
pr_debug("Entry used bits: ");
print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits,
sizeof(used_bits), false);
}
@@ -643,8 +645,10 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test,
const int *ids, const int *users,
const int *ssids)
{
KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs);
KUNIT_ASSERT_EQ(test, invs->num_invs, num_invs);
KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes);
KUNIT_EXPECT_TRUE(test, invs->has_range_inv);
KUNIT_EXPECT_TRUE(test, invs->has_full_cont_range_inv);
while (num_invs--) {
KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]);
KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users),
@@ -654,38 +658,43 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test,
}
static struct arm_smmu_invs invs1 = {
.max_invs = 3,
.num_invs = 3,
.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, },
{ .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, },
{ .type = INV_TYPE_ATS, .id = 3, }, },
.inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, },
{ .smmu = &smmu, .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, },
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 3, }, },
};
static struct arm_smmu_invs invs2 = {
.max_invs = 3,
.num_invs = 3,
.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
{ .type = INV_TYPE_ATS, .id = 4, },
{ .type = INV_TYPE_ATS, .id = 5, }, },
.inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 4, },
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, },
};
static struct arm_smmu_invs invs3 = {
.max_invs = 3,
.num_invs = 3,
.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
{ .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */
{ .type = INV_TYPE_ATS, .id = 6, }, },
.inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 6, }, },
};
static struct arm_smmu_invs invs4 = {
.max_invs = 3,
.num_invs = 3,
.inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 },
{ .type = INV_TYPE_ATS, .id = 10, .ssid = 3 },
{ .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, },
.inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 1 },
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 },
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, },
};
static struct arm_smmu_invs invs5 = {
.max_invs = 3,
.num_invs = 3,
.inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 },
{ .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */
{ .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, },
.inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 2 },
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */
{ .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, },
};
static void arm_smmu_v3_invs_test(struct kunit *test)
@@ -704,17 +713,22 @@ static void arm_smmu_v3_invs_test(struct kunit *test)
/* New array */
test_a = arm_smmu_invs_alloc(0);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a);
KUNIT_EXPECT_EQ(test, test_a->num_invs, 0);
KUNIT_EXPECT_FALSE(test, test_a->has_range_inv);
KUNIT_EXPECT_FALSE(test, test_a->has_full_cont_range_inv);
/* Test1: merge invs1 (new array) */
test_b = arm_smmu_invs_merge(test_a, &invs1);
kfree(test_a);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b);
arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0,
results1[0], results1[1], results1[2]);
/* Test2: merge invs2 (new array) */
test_a = arm_smmu_invs_merge(test_b, &invs2);
kfree(test_b);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a);
arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0,
results2[0], results2[1], results2[2]);
@@ -726,6 +740,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test)
/* Test4: merge invs3 (new array) */
test_b = arm_smmu_invs_merge(test_a, &invs3);
kfree(test_a);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b);
arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0,
results4[0], results4[1], results4[2]);
@@ -737,6 +752,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test)
/* Test6: purge test_b (new array) */
test_a = arm_smmu_invs_purge(test_b);
kfree(test_b);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a);
arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0,
results6[0], results6[1], results6[2]);
@@ -748,12 +764,14 @@ static void arm_smmu_v3_invs_test(struct kunit *test)
/* Test8: merge invs4 (new array) */
test_b = arm_smmu_invs_merge(test_a, &invs4);
kfree(test_a);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b);
arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0,
results7[0], results7[1], results7[2]);
/* Test9: merge invs5 (new array) */
test_a = arm_smmu_invs_merge(test_b, &invs5);
kfree(test_b);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_a);
arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0,
results8[0], results8[1], results8[2]);
@@ -765,6 +783,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test)
/* Test11: purge test_a (new array) */
test_b = arm_smmu_invs_purge(test_a);
kfree(test_a);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, test_b);
arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0,
results10[0], results10[1], results10[2]);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+450 -178
View File
@@ -40,6 +40,11 @@ module_param(disable_msipolling, bool, 0444);
MODULE_PARM_DESC(disable_msipolling,
"Disable MSI-based polling for CMD_SYNC completion.");
static unsigned int cmdq_max_n_shift = CMDQ_MAX_SZ_SHIFT;
module_param(cmdq_max_n_shift, uint, 0444);
MODULE_PARM_DESC(cmdq_max_n_shift,
"Cap on the command queue depth, as log2 of the number of entries. Defaults to the hardware maximum; the queue never shrinks below one page. A kdump kernel always uses one page.");
static const struct iommu_ops arm_smmu_ops;
static struct iommu_dirty_ops arm_smmu_dirty_ops;
@@ -91,6 +96,7 @@ DEFINE_MUTEX(arm_smmu_asid_lock);
static struct arm_smmu_option_prop arm_smmu_options[] = {
{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
{ ARM_SMMU_OPT_OVR_INSTCFG_DATA, "arm,instdata-override"},
{ 0, NULL},
};
@@ -181,8 +187,8 @@ static void queue_sync_cons_out(struct arm_smmu_queue *q)
static void queue_inc_cons(struct arm_smmu_ll_queue *q)
{
u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
u32 cons = Q_POS(q, q->cons) + 1;
q->cons = Q_OVF(q->cons) | Q_POS(q, cons);
}
static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
@@ -192,8 +198,7 @@ static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
return;
llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
Q_IDX(llq, llq->cons);
llq->cons = Q_OVF(llq->prod) | Q_POS(llq, llq->cons);
queue_sync_cons_out(q);
}
@@ -218,8 +223,8 @@ static int queue_sync_prod_in(struct arm_smmu_queue *q)
static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
{
u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
u32 prod = Q_POS(q, q->prod) + n;
return Q_OVF(q->prod) | Q_POS(q, prod);
}
static void queue_poll_init(struct arm_smmu_device *smmu,
@@ -1053,6 +1058,19 @@ static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l,
return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]);
}
static void arm_smmu_invs_update_caps(struct arm_smmu_invs *invs,
const struct arm_smmu_inv *inv)
{
if (arm_smmu_inv_is_ats(inv))
invs->has_ats = true;
if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
invs->has_range_inv = true;
if (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)
invs->has_full_cont_range_inv = true;
}
}
/**
* arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for
* arm_smmu_invs_merge() or arm_smmu_invs_unref()
@@ -1123,8 +1141,7 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
*/
if (new != new_invs->inv)
WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1);
if (arm_smmu_inv_is_ats(new))
new_invs->has_ats = true;
arm_smmu_invs_update_caps(new_invs, new);
new++;
}
@@ -1234,8 +1251,7 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs)
arm_smmu_invs_for_each_entry(invs, i, inv) {
new_invs->inv[num_invs] = *inv;
if (arm_smmu_inv_is_ats(inv))
new_invs->has_ats = true;
arm_smmu_invs_update_caps(new_invs, inv);
num_invs++;
}
@@ -1271,7 +1287,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
STRTAB_STE_1_EATS | STRTAB_STE_1_MEV |
STRTAB_STE_1_INSTCFG);
used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
/*
@@ -1287,7 +1304,8 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
if (cfg & BIT(1)) {
used_bits[1] |=
cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV |
STRTAB_STE_1_INSTCFG);
used_bits[2] |=
cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
@@ -1515,6 +1533,9 @@ static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
{
u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
/* Ensure the zero-cleared L2 table is fully visible. */
dma_wmb();
/* The HW has 64 bit atomicity with stores to the L2 CD table */
WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
}
@@ -1804,6 +1825,9 @@ static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
/* Ensure the new L2 table is fully visible. */
dma_wmb();
/* The HW has 64 bit atomicity with stores to the L2 STE table */
WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
}
@@ -1900,7 +1924,11 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
STRTAB_STE_1_S1STALLD :
0) |
FIELD_PREP(STRTAB_STE_1_EATS,
ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) |
FIELD_PREP(STRTAB_STE_1_INSTCFG,
smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ?
STRTAB_STE_1_INSTCFG_DATA :
STRTAB_STE_1_INSTCFG_INCOMING));
if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
s1dss == STRTAB_STE_1_S1DSS_BYPASS)
@@ -1952,7 +1980,11 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
target->data[1] = cpu_to_le64(
FIELD_PREP(STRTAB_STE_1_EATS,
ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) |
FIELD_PREP(STRTAB_STE_1_INSTCFG,
smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA ?
STRTAB_STE_1_INSTCFG_DATA :
STRTAB_STE_1_INSTCFG_INCOMING));
if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
@@ -2363,8 +2395,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
return IRQ_WAKE_THREAD;
}
static struct arm_smmu_cmd
arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid,
struct arm_smmu_tlbi *tlbi)
{
size_t log2_span;
size_t span_mask;
@@ -2386,8 +2418,8 @@ arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
* This has the unpleasant side-effect of invalidating all PASID-tagged
* ATC entries within the address range.
*/
page_start = iova >> inval_grain_shift;
page_end = (iova + size - 1) >> inval_grain_shift;
page_start = tlbi->iova >> inval_grain_shift;
page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift;
/*
* In an ATS Invalidate Request, the address must be aligned on the
@@ -2460,126 +2492,231 @@ static void arm_smmu_tlb_inv_context(void *cookie)
arm_smmu_domain_inv(smmu_domain);
}
static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd, bool leaf,
unsigned long iova, size_t size,
size_t granule, size_t pgsize)
/*
* Check address alignment for TTL hint per SMMUv3 H.a Section 4.4.1. Address
* bits below the alignment must be zero, otherwise UNPREDICTABLE.
*/
static bool arm_smmu_ttl_addr_aligned(u64 address, unsigned int tg,
unsigned int ttl)
{
unsigned long end = iova + size, num_pages = 0, tg = pgsize;
u64 orig_data0 = cmd->data[0];
size_t inv_range = granule;
u8 ttl = 0, tg_enc = 0;
unsigned int pgsz_lg2 = arm_smmu_pt_level_to_lg2sz(tg, 3 - ttl);
if (WARN_ON_ONCE(!size))
return;
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
num_pages = size >> tg;
/* Convert page size of 12,14,16 (log2) to 1,2,3 */
tg_enc = (tg - 10) / 2;
/*
* Determine what level the granule is at. For non-leaf, both
* io-pgtable and SVA pass a nominal last-level granule because
* they don't know what level(s) actually apply, so ignore that
* and leave TTL=0. However for various errata reasons we still
* want to use a range command, so avoid the SVA corner case
* where both scale and num could be 0 as well.
*/
if (leaf)
ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
num_pages++;
}
while (iova < end) {
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
/*
* On each iteration of the loop, the range is 5 bits
* worth of the aligned size remaining.
* The range in pages is:
*
* range = (num_pages & (0x1f << __ffs(num_pages)))
*/
unsigned long scale, num;
/* Determine the power of 2 multiple number of pages */
scale = __ffs(num_pages);
/* Determine how many chunks of 2^scale size we have */
num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
/* Keep the pre-DS 5-bit truncation when scale > 31 */
cmd->data[0] = orig_data0 |
FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale & 0x1f);
/* range is num * 2^scale * pgsize */
inv_range = num << (scale + tg);
/* Clear out the lower order bits for the next iteration */
num_pages -= num << scale;
}
/*
* IPA has fewer bits than VA, but they are reserved in the
* command and something would be very broken if iova had them
* set.
*/
cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) |
FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
(iova & ~GENMASK_U64(11, 0));
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
iova += inv_range;
}
return !(address & GENMASK_U64(pgsz_lg2 - 1, 0));
}
static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
size_t granule)
struct arm_smmu_range_inv {
u64 start_tg;
/* Normal integer, not encoded. 0 means 0.*/
u64 num;
unsigned int scale;
};
static unsigned int arm_smmu_range_inv_calc_scale(u64 num_tg)
{
size_t max_tlbi_ops;
return fls64((num_tg - 1) / (CMDQ_TLBI_RANGE_NUM_MAX + 1));
}
/* 0 size means invalidate all */
if (!size || size == SIZE_MAX)
return true;
static u64 arm_smmu_range_inv_calc_num(u64 num_tg, unsigned int scale)
{
return DIV_ROUND_UP_ULL(num_tg, 1ULL << scale);
}
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV)
return false;
/*
* Initialize the smallest range invalidation covering num_tg and ending at
* last_tg.
*/
static struct arm_smmu_range_inv arm_smmu_range_inv_init_end(u64 last_tg,
u64 num_tg)
{
struct arm_smmu_range_inv range_inv = {};
if (!num_tg)
return range_inv;
range_inv.scale = arm_smmu_range_inv_calc_scale(num_tg);
range_inv.num = arm_smmu_range_inv_calc_num(num_tg, range_inv.scale);
range_inv.start_tg = last_tg - ((range_inv.num << range_inv.scale) - 1);
return range_inv;
}
static void
arm_smmu_tlbi_add_range_cmd(struct arm_smmu_tlbi *tlbi,
const struct arm_smmu_range_inv *range_inv, u8 ttl,
u8 tg_enc)
{
struct arm_smmu_cmd *cmd = &tlbi->range.cmds[tlbi->range.num_cmds++];
unsigned int tgsz_lg2 = tg_enc * 2 + 10;
u64 iova = range_inv->start_tg << tgsz_lg2;
unsigned int num = range_inv->num - 1;
/* 16K granule TTL=1 is reserved (Section 4.4.1) */
if (WARN_ON(tgsz_lg2 == 14 && ttl == 1))
ttl = 0;
/* Verify address alignment for the TTL hint */
if (ttl && !arm_smmu_ttl_addr_aligned(iova, tgsz_lg2, ttl))
ttl = 0;
/*
* Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h,
* this is used as a threshold to replace "size_opcode" commands with a
* single "nsize_opcode" command, when SMMU doesn't implement the range
* invalidation feature, where there can be too many per-granule TLBIs,
* resulting in a soft lockup.
* SMMUv3 H.a Section 4.4.1: TG!=0, NUM==0, SCALE==0, TTL==0 is Reserved
* and causes CERROR_ILL. Single tg uses NUM=0, SCALE=0 with a TTL hint
* to target only the exact leaf entry.
*
* For a single tg invalidation a 0 TTL can come from places like the
* SVA path that don't have enough information to get a TTL, or as a
* side of effect of the splitting.
*
* A single-TG invalidation cannot reach this point if it is part of a
* CONT group, so it is safe to transform it into a single invalidation.
* The ARM_SMMU_OPT_FULL_CONT_RANGE_INV errata does not apply.
*/
max_tlbi_ops = 1 << (ilog2(granule) - 3);
return size >= max_tlbi_ops * granule;
if (!num && !range_inv->scale && !ttl)
tg_enc = 0;
cmd->data[0] = FIELD_PREP(CMDQ_TLBI_0_NUM, num) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, range_inv->scale);
cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | iova;
}
/* Used by non INV_TYPE_ATS* invalidations */
static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
bool leaf,
unsigned long iova, size_t size,
unsigned int granule)
/*
* Generate up to two range TLBI command payloads covering [iova, iova+size).
* Sets use_full_inv if the range is too large to represent.
*
* Normally the first range invalidation is the largest representable span which
* does not exceed the requested range. If necessary, the second range
* invalidation is the smallest representable range covering the remainder and
* is anchored at the end. Any excess coverage from the second range
* invalidation overlaps the first instead of exceeding the requested range.
*
* For SVA on an invs containing an SMMU with ARM_SMMU_OPT_FULL_CONT_RANGE_INV,
* produce only a single range invalidation and overinvalidate so any potential CONT is
* covered by one command.
*/
static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi,
bool single_range_inv)
{
if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) {
struct arm_smmu_cmd nsize_cmd = *cmd;
u8 tgsz_lg2 = tlbi->tgsz_lg2;
struct arm_smmu_range_inv first = { .start_tg = tlbi->iova >>
tgsz_lg2 };
u64 last_tg = (tlbi->iova + tlbi->size - 1) >> tgsz_lg2;
u64 num_tg = last_tg - first.start_tg + 1;
u8 tg_enc = (tgsz_lg2 - 10) / 2;
struct arm_smmu_range_inv trail;
u8 ttl = 0;
u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode,
CMDQ_0_OP);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd);
/*
* Determine what level the granule is at. For non-leaf, both io-pgtable
* and SVA pass a nominal last-level granule because they don't know
* what level(s) actually apply, so leave TTL=0.
*/
if (tlbi->leaf_only)
ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tgsz_lg2 - 3));
/*
* The spec defines the invalidated range as:
* Range = ((NUM+1) * 2^SCALE) * Translation_Granule_Size
* NUM is 5 bits, so (NUM+1) covers 1..32 granules. Find the smallest
* SCALE at which a single command could cover num_tg.
*
* Unlike other IOMMUs the spec has no alignment requirement on the
* address beyond alignment to tg (so long as TTL=0).
*/
first.scale = arm_smmu_range_inv_calc_scale(num_tg);
if (first.scale > 31) {
/* Range too large for a single command do full invalidation */
tlbi->range.use_full_inv = true;
return;
}
arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf,
iova, size, granule, inv->pgsize);
if (single_range_inv) {
/*
* Produce a single invalidation by rounding up and disabling
* the trailer.
*/
first.num = arm_smmu_range_inv_calc_num(num_tg, first.scale);
trail.num = 0;
} else {
/*
* Produce two invalidations by rounding down and adding a
* second trailing range invalidation anchored at the end.
*/
first.num = num_tg >> first.scale;
trail = arm_smmu_range_inv_init_end(
last_tg, num_tg - ((u64)first.num << first.scale));
}
arm_smmu_tlbi_add_range_cmd(tlbi, &first, ttl, tg_enc);
if (trail.num)
arm_smmu_tlbi_add_range_cmd(tlbi, &trail, ttl, tg_enc);
}
/*
* One TLBI command per IOTLB entry, assuming the entries are all at least
* iopte_granule sized. Sets use_full_inv if too many commands would be needed
* which indicates too high a latency. The threshold is similar to MAX_DVM_OPS
* in arch/arm64/include/asm/tlbflush.h for the 4k PAGE_SIZE.
*/
static void arm_smmu_tlbi_calc_single(struct arm_smmu_tlbi *tlbi)
{
unsigned long num_ops = tlbi->size / tlbi->iopte_size;
if (!num_ops || num_ops > 512) {
tlbi->single.use_full_inv = true;
return;
}
tlbi->single.num = num_ops;
}
static void arm_smmu_inv_all_cmd(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
u64p_replace_bits(&cmd->data[0], inv->nsize_opcode, CMDQ_0_OP);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd);
}
/*
* Used by non INV_TYPE_ATS* invalidations. Returns true if it fell back to
* full invalidation using nsize_opcode.
*/
static bool arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
struct arm_smmu_tlbi *tlbi)
{
u64 iova = tlbi->iova;
unsigned int i;
if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
if (tlbi->range.use_full_inv) {
arm_smmu_inv_all_cmd(inv, cmds, cmd);
return true;
}
for (i = 0; i < tlbi->range.num_cmds; i++) {
struct arm_smmu_cmd range_cmd = tlbi->range.cmds[i];
range_cmd.data[0] |= cmd->data[0];
range_cmd.data[1] |= cmd->data[1];
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds,
&range_cmd);
}
return false;
}
if (tlbi->single.use_full_inv) {
arm_smmu_inv_all_cmd(inv, cmds, cmd);
return true;
}
for (i = 0; i < tlbi->single.num; i++) {
cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
(iova & ~GENMASK_U64(11, 0));
iova += tlbi->iopte_size;
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, cmd);
}
return false;
}
static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
@@ -2598,10 +2735,10 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
return false;
}
static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
unsigned long iova, size_t size,
unsigned int granule, bool leaf)
static void arm_smmu_domain_tlbi_inv(struct arm_smmu_tlbi *tlbi,
struct arm_smmu_invs *invs)
{
struct arm_smmu_inv *used_s12_vmall = NULL;
struct arm_smmu_cmdq_batch cmds = {};
struct arm_smmu_inv *cur;
struct arm_smmu_inv *end;
@@ -2630,18 +2767,24 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
case INV_TYPE_S1_ASID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi);
break;
case INV_TYPE_S2_VMID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
0, cur->id);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0,
cur->id);
if (arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi))
used_s12_vmall = cur + 1;
break;
case INV_TYPE_S2_VMID_S1_CLEAR:
/* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
if (arm_smmu_inv_size_too_big(cur->smmu, size, granule))
/*
* S2_VMID used CMDQ_OP_TLBI_S12_VMALL which already
* flushed S1 entries. These two types always come in
* pairs and arm_smmu_inv_cmp() ensures that they are
* consecutive in the list for the same SMMU. There may
* be several pairings so check this is paired with the
* one that did the full invalidation.
*/
if (used_s12_vmall == cur)
break;
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
@@ -2652,7 +2795,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid,
iova, size));
tlbi));
break;
case INV_TYPE_ATS_FULL:
arm_smmu_cmdq_batch_add_cmd(
@@ -2679,12 +2822,20 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
}
}
void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long iova, size_t size,
unsigned int granule, bool leaf)
/*
* Perform the invalidation with the parameters described by tlbi on everything
* caching for smmu_domain. tlbi is an on-stack structure that contains both the
* input description and working memory for this function it must be discarded
* after this returns.
*/
void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi,
struct arm_smmu_domain *smmu_domain)
{
struct arm_smmu_invs *invs;
if (!tlbi->single.use_full_inv)
arm_smmu_tlbi_calc_single(tlbi);
/*
* An invalidation request must follow some IOPTE change and then load
* an invalidation array. In the meantime, a domain attachment mutates
@@ -2702,7 +2853,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
*
* [CPU0] | [CPU1]
* change IOPTE on new domain: |
* arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs()
* arm_smmu_domain_tlbi() { | arm_smmu_install_new_domain_invs()
* smp_mb(); // ensures IOPTE | arm_smmu_install_ste_for_dev {
* // seen by SMMU | dma_wmb(); // ensures invs update
* // load the updated invs | // before updating STE
@@ -2715,6 +2866,20 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
rcu_read_lock();
invs = rcu_dereference(smmu_domain->invs);
/*
* Only precompute range invalidation commands when they will be used.
* The invs generation ensures this matches the instances being
* invalidated.
*/
if (invs->has_range_inv) {
if (!tlbi->range.use_full_inv) {
arm_smmu_tlbi_calc_range(
tlbi,
smmu_domain->stage == ARM_SMMU_DOMAIN_SVA &&
invs->has_full_cont_range_inv);
}
}
/*
* Avoid locking unless ATS is being used. No ATC invalidation can be
* going on after a domain is detached.
@@ -2723,10 +2888,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long flags;
read_lock_irqsave(&invs->rwlock, flags);
__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
arm_smmu_domain_tlbi_inv(tlbi, invs);
read_unlock_irqrestore(&invs->rwlock, flags);
} else {
__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
arm_smmu_domain_tlbi_inv(tlbi, invs);
}
rcu_read_unlock();
@@ -2742,12 +2907,23 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
iommu_iotlb_gather_add_page(domain, gather, iova, granule);
}
/*
* Called by io-pgtable-arm.c for each single table level it wants to remove.
* size is the size of the table level and granule is the tg in bytes. This must
* clear the walk cache and any leaves within the range.
*/
static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_tlbi tlbi = {
.tgsz_lg2 = smmu_domain->tgsz_lg2,
.iova = iova,
.size = size,
.iopte_size = 1 << smmu_domain->tgsz_lg2,
};
arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false);
arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static const struct iommu_flush_ops arm_smmu_flush_ops = {
@@ -2933,6 +3109,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
return -ENOMEM;
smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
smmu_domain->tgsz_lg2 = __ffs(pgtbl_cfg.pgsize_bitmap);
smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
smmu_domain->domain.geometry.force_aperture = true;
if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
@@ -3172,15 +3349,13 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
static struct arm_smmu_inv *
arm_smmu_master_build_inv(struct arm_smmu_master *master,
enum arm_smmu_inv_type type, u32 id, ioasid_t ssid,
size_t pgsize)
enum arm_smmu_inv_type type, u32 id, ioasid_t ssid)
{
struct arm_smmu_invs *build_invs = master->build_invs;
struct arm_smmu_inv *cur, inv = {
.smmu = master->smmu,
.type = type,
.id = id,
.pgsize = pgsize,
};
if (WARN_ON(build_invs->num_invs >= build_invs->max_invs))
@@ -3232,28 +3407,24 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled,
ioasid_t ssid, struct arm_smmu_domain *smmu_domain)
{
const bool nesting = smmu_domain->nest_parent;
size_t pgsize = 0, i;
size_t i;
iommu_group_mutex_assert(master->dev);
master->build_invs->num_invs = 0;
/* Range-based invalidation requires the leaf pgsize for calculation */
if (master->smmu->features & ARM_SMMU_FEAT_RANGE_INV)
pgsize = __ffs(smmu_domain->domain.pgsize_bitmap);
switch (smmu_domain->stage) {
case ARM_SMMU_DOMAIN_SVA:
case ARM_SMMU_DOMAIN_S1:
if (!arm_smmu_master_build_inv(master, INV_TYPE_S1_ASID,
smmu_domain->cd.asid,
IOMMU_NO_PASID, pgsize))
IOMMU_NO_PASID))
return NULL;
break;
case ARM_SMMU_DOMAIN_S2:
if (!arm_smmu_master_build_inv(master, INV_TYPE_S2_VMID,
smmu_domain->s2_cfg.vmid,
IOMMU_NO_PASID, pgsize))
IOMMU_NO_PASID))
return NULL;
break;
default:
@@ -3265,7 +3436,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled,
if (nesting) {
if (!arm_smmu_master_build_inv(
master, INV_TYPE_S2_VMID_S1_CLEAR,
smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID, 0))
smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID))
return NULL;
}
@@ -3276,7 +3447,7 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled,
*/
if (!arm_smmu_master_build_inv(
master, nesting ? INV_TYPE_ATS_FULL : INV_TYPE_ATS,
master->streams[i].id, ssid, 0))
master->streams[i].id, ssid))
return NULL;
}
@@ -3285,9 +3456,9 @@ arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled,
return master->build_invs;
}
static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
struct iommu_domain *domain,
ioasid_t ssid)
static struct arm_smmu_master_domain *
arm_smmu_remove_master_domain(struct arm_smmu_master *master,
struct iommu_domain *domain, ioasid_t ssid)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
struct arm_smmu_master_domain *master_domain;
@@ -3295,7 +3466,7 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
unsigned long flags;
if (!smmu_domain)
return;
return NULL;
if (domain->type == IOMMU_DOMAIN_NESTED)
nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
@@ -3310,8 +3481,24 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
}
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
/* arm_smmu_attach_release() will free it */
return master_domain;
}
/* Release the old master_domain detached by arm_smmu_remove_master_domain() */
void arm_smmu_attach_release(struct arm_smmu_attach_state *state)
{
struct arm_smmu_master_domain *master_domain = state->old_master_domain;
struct arm_smmu_master *master = state->master;
iommu_group_mutex_assert(master->dev);
if (!master_domain)
return;
arm_smmu_disable_iopf(master, master_domain);
kfree(master_domain);
state->old_master_domain = NULL;
}
/*
@@ -3609,7 +3796,8 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
}
arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
state->old_master_domain = arm_smmu_remove_master_domain(
master, state->old_domain, state->ssid);
arm_smmu_install_old_domain_invs(state);
master->ats_enabled = state->ats_enabled;
}
@@ -3684,6 +3872,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev,
arm_smmu_attach_commit(&state);
mutex_unlock(&arm_smmu_asid_lock);
arm_smmu_attach_release(&state);
return 0;
}
@@ -3766,8 +3955,10 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
mutex_lock(&arm_smmu_asid_lock);
ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
if (ret)
goto out_unlock;
if (ret) {
mutex_unlock(&arm_smmu_asid_lock);
return ret;
}
/*
* We don't want to obtain to the asid_lock too early, so fix up the
@@ -3781,10 +3972,9 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
arm_smmu_attach_commit(&state);
out_unlock:
mutex_unlock(&arm_smmu_asid_lock);
return ret;
arm_smmu_attach_release(&state);
return 0;
}
static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
@@ -3804,9 +3994,11 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
arm_smmu_clear_cd(master, pasid);
if (master->ats_enabled)
arm_smmu_atc_inv_master(master, pasid);
arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
state.old_master_domain = arm_smmu_remove_master_domain(
master, &smmu_domain->domain, pasid);
arm_smmu_install_old_domain_invs(&state);
mutex_unlock(&arm_smmu_asid_lock);
arm_smmu_attach_release(&state);
/*
* When the last user of the CD table goes away downgrade the STE back
@@ -3869,6 +4061,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
arm_smmu_install_ste_for_dev(master, ste);
arm_smmu_attach_commit(&state);
mutex_unlock(&arm_smmu_asid_lock);
arm_smmu_attach_release(&state);
/*
* This has to be done after removing the master from the
@@ -4025,13 +4218,18 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_tlbi tlbi = {
.tgsz_lg2 = smmu_domain->tgsz_lg2,
.iova = gather->start,
.size = gather->end - gather->start + 1,
.iopte_size = gather->pgsize,
.leaf_only = true,
};
if (!gather->pgsize)
return;
arm_smmu_domain_inv_range(smmu_domain, gather->start,
gather->end - gather->start + 1,
gather->pgsize, true);
arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
static phys_addr_t
@@ -4122,6 +4320,13 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
sizeof(master->streams[0]), arm_smmu_stream_id_cmp,
NULL);
/*
* Clear after sorting: RB_CLEAR_NODE() records the node's own address,
* which sort_nonatomic() invalidates by relocating the entries.
*/
for (i = 0; i < fwspec->num_ids; i++)
RB_CLEAR_NODE(&master->streams[i].node);
mutex_lock(&smmu->streams_mutex);
for (i = 0; i < fwspec->num_ids; i++) {
struct arm_smmu_stream *new_stream = &master->streams[i];
@@ -4154,7 +4359,9 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
if (ret) {
for (i--; i >= 0; i--)
rb_erase(&master->streams[i].node, &smmu->streams);
if (!RB_EMPTY_NODE(&master->streams[i].node))
rb_erase(&master->streams[i].node,
&smmu->streams);
kfree(master->streams);
kfree(master->build_invs);
}
@@ -4174,7 +4381,8 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master)
mutex_lock(&smmu->streams_mutex);
for (i = 0; i < fwspec->num_ids; i++)
rb_erase(&master->streams[i].node, &smmu->streams);
if (!RB_EMPTY_NODE(&master->streams[i].node))
rb_erase(&master->streams[i].node, &smmu->streams);
mutex_unlock(&smmu->streams_mutex);
kfree(master->streams);
@@ -4412,6 +4620,56 @@ static struct iommu_dirty_ops arm_smmu_dirty_ops = {
};
/* Probing and initialisation functions */
/**
* arm_smmu_queue_max_n_shift() - pick the log2 depth of a queue
* @hw_max_n_shift: log2 depth the hardware advertises in IDR1
* @ent_sz_shift: log2 of the queue entry size in bytes
* @limit_n_shift: log2 depth to cap the queue at
*
* A kdump capture kernel gets one page worth of entries whatever the limit.
*
* @limit_n_shift is floored at one page, because coherent DMA is page
* granular: a shallower queue occupies the same memory as one that fills the
* page, and arm_smmu_init_one_queue() stops shrinking at a page too.
*/
static u32 arm_smmu_queue_max_n_shift(u32 hw_max_n_shift, u32 ent_sz_shift,
u32 limit_n_shift)
{
u32 floor_n_shift = PAGE_SHIFT - ent_sz_shift;
if (is_kdump_kernel())
return min(hw_max_n_shift, floor_n_shift);
limit_n_shift = max(limit_n_shift, floor_n_shift);
return min(hw_max_n_shift, limit_n_shift);
}
static inline u32 arm_smmu_evtq_max_n_shift(u32 hw_max_n_shift)
{
/* Capped to ensure natural alignment */
return arm_smmu_queue_max_n_shift(hw_max_n_shift, EVTQ_ENT_SZ_SHIFT,
EVTQ_MAX_SZ_SHIFT);
}
static inline u32 arm_smmu_priq_max_n_shift(u32 hw_max_n_shift)
{
/* Capped to ensure natural alignment */
return arm_smmu_queue_max_n_shift(hw_max_n_shift, PRIQ_ENT_SZ_SHIFT,
PRIQ_MAX_SZ_SHIFT);
}
/*
* Command queues are also allocated by the Tegra241 CMDQV for its VCMDQs, which
* need the same depth decision.
*/
u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift)
{
/* Capped to ensure natural alignment */
return arm_smmu_queue_max_n_shift(hw_max_n_shift, CMDQ_ENT_SZ_SHIFT,
min(CMDQ_MAX_SZ_SHIFT, cmdq_max_n_shift));
}
int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
struct arm_smmu_queue *q, void __iomem *page,
unsigned long prod_off, unsigned long cons_off,
@@ -5009,12 +5267,21 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
/* Arm errata 2268618, 2812531 */
smmu->features &= ~ARM_SMMU_FEAT_NESTING;
}
/* Arm errata 3777127 */
smmu->options |= ARM_SMMU_OPT_FULL_CONT_RANGE_INV;
break;
case IIDR_PRODUCTID_ARM_MMU_L1:
case IIDR_PRODUCTID_ARM_MMU_S3:
/* Arm errata 3878312/3995052 */
/* Arm errata 3878312 */
smmu->features &= ~ARM_SMMU_FEAT_BTM;
break;
case IIDR_PRODUCTID_ARM_MMU_S3:
/* Arm errata 3995052 */
smmu->features &= ~ARM_SMMU_FEAT_BTM;
/* Arm errata 3673557 */
if (variant < 1 || (variant == 1 && revision < 1))
smmu->options |=
ARM_SMMU_OPT_FULL_CONT_RANGE_INV;
break;
}
break;
}
@@ -5155,9 +5422,14 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
if (reg & IDR1_ATTR_TYPES_OVR)
smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
/* Queue sizes, capped to ensure natural alignment */
smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
FIELD_GET(IDR1_CMDQS, reg));
if ((smmu->options & ARM_SMMU_OPT_OVR_INSTCFG_DATA) &&
!(reg & IDR1_ATTR_PERMS_OVR)) {
dev_err(smmu->dev, "Inst/Data attribute override not supported\n");
return -ENXIO;
}
smmu->cmdq.q.llq.max_n_shift =
arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, reg));
if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
/*
* We don't support splitting up batches, so one batch of
@@ -5170,10 +5442,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
return -ENXIO;
}
smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
FIELD_GET(IDR1_EVTQS, reg));
smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
FIELD_GET(IDR1_PRIQS, reg));
smmu->evtq.q.llq.max_n_shift =
arm_smmu_evtq_max_n_shift(FIELD_GET(IDR1_EVTQS, reg));
smmu->priq.q.llq.max_n_shift =
arm_smmu_priq_max_n_shift(FIELD_GET(IDR1_PRIQS, reg));
/* SID/SSID sizes */
smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
@@ -5633,7 +5905,7 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev)
{
struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
arm_smmu_device_disable(smmu);
arm_smmu_disable_action(smmu);
}
static const struct of_device_id arm_smmu_of_match[] = {
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+69 -8
View File
@@ -52,6 +52,7 @@ struct arm_vsmmu;
#define IDR1_QUEUES_PRESET (1 << 29)
#define IDR1_REL (1 << 28)
#define IDR1_ATTR_TYPES_OVR (1 << 27)
#define IDR1_ATTR_PERMS_OVR (1 << 26)
#define IDR1_CMDQS GENMASK(25, 21)
#define IDR1_EVTQS GENMASK(20, 16)
#define IDR1_PRIQS GENMASK(15, 11)
@@ -187,6 +188,8 @@ struct arm_vsmmu;
#define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
#define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
/* A position is Q_WRP | Q_IDX, wrapping at twice the queue capacity */
#define Q_POS(llq, p) (Q_WRP(llq, p) | Q_IDX(llq, p))
#define Q_OVERFLOW_FLAG (1U << 31)
#define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
#define Q_ENT(q, p) ((q)->base + \
@@ -285,6 +288,10 @@ static inline u32 arm_smmu_strtab_l2_idx(u32 sid)
#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
#define STRTAB_STE_1_SHCFG_INCOMING 1UL
#define STRTAB_STE_1_INSTCFG GENMASK_ULL(51, 50)
#define STRTAB_STE_1_INSTCFG_INCOMING 0UL
#define STRTAB_STE_1_INSTCFG_DATA 2UL
#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
@@ -736,10 +743,9 @@ struct arm_smmu_inv {
u8 size_opcode;
u8 nsize_opcode;
u32 id; /* ASID or VMID or SID */
union {
size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */
u32 ssid; /* INV_TYPE_ATS */
};
/* Only used by INV_TYPE_ATS */
u32 ssid;
int users; /* users=0 to mark as a trash to be purged */
};
@@ -759,6 +765,9 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv)
* Must not be greater than @num_invs
* @rwlock: optional rwlock to fence ATS operations
* @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL
* @has_range_inv: flag if any entry's SMMU supports range invalidation
* @has_full_cont_range_inv: flag if any entry's SMMU requires the CONT range
* invalidation workaround
* @rcu: rcu head for kfree_rcu()
* @inv: flexible invalidation array
*
@@ -788,6 +797,8 @@ struct arm_smmu_invs {
size_t num_trashes;
rwlock_t rwlock;
bool has_ats;
bool has_range_inv;
bool has_full_cont_range_inv;
struct rcu_head rcu;
struct arm_smmu_inv inv[] __counted_by(max_invs);
};
@@ -805,6 +816,40 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
return new_invs;
}
/* Generic page-table level 0 is the leaf-only level. */
static inline unsigned int arm_smmu_pt_level_to_lg2sz(unsigned int tgsz_lg2,
unsigned int level)
{
return tgsz_lg2 + (tgsz_lg2 - ilog2(sizeof(u64))) * level;
}
static inline unsigned int arm_smmu_pt_lg2sz_to_level(unsigned int tgsz_lg2,
unsigned int lg2sz)
{
return (lg2sz - tgsz_lg2) / (tgsz_lg2 - ilog2(sizeof(u64)));
}
struct arm_smmu_tlbi {
unsigned long iova;
size_t size;
/* page or block size of the leaf iopte */
unsigned int iopte_size;
/* Base Translation Granule of the page table */
u8 tgsz_lg2;
bool leaf_only;
struct {
bool use_full_inv;
u16 num;
} single;
struct {
bool use_full_inv;
u8 num_cmds;
struct arm_smmu_cmd cmds[2];
} range;
};
struct arm_smmu_evtq {
struct arm_smmu_queue q;
struct iopf_queue *iopf;
@@ -934,6 +979,12 @@ struct arm_smmu_device {
#define ARM_SMMU_OPT_MSIPOLL (1 << 2)
#define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3)
#define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4)
#define ARM_SMMU_OPT_OVR_INSTCFG_DATA (1 << 5)
/*
* Range invalidation is mandatory and one range invalidation must fully span an
* invalidated CONT
*/
#define ARM_SMMU_OPT_FULL_CONT_RANGE_INV (1 << 6)
u32 options;
struct arm_smmu_cmdq cmdq;
@@ -1049,6 +1100,7 @@ struct arm_smmu_domain {
spinlock_t devices_lock;
bool enforce_cache_coherency : 1;
bool nest_parent : 1;
u8 tgsz_lg2;
struct mmu_notifier mmu_notifier;
};
@@ -1154,17 +1206,24 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
struct arm_smmu_cd *cd, struct iommu_domain *old);
void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long iova, size_t size,
unsigned int granule, bool leaf);
void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi,
struct arm_smmu_domain *smmu_domain);
static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain)
{
arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false);
/* Prefilled for invalidate all */
struct arm_smmu_tlbi tlbi = {
.tgsz_lg2 = smmu_domain->tgsz_lg2,
.single.use_full_inv = true,
.range.use_full_inv = true,
};
arm_smmu_domain_tlbi(&tlbi, smmu_domain);
}
void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq);
u32 arm_smmu_cmdq_max_n_shift(u32 hw_max_n_shift);
int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
struct arm_smmu_queue *q, void __iomem *page,
unsigned long prod_off, unsigned long cons_off,
@@ -1204,12 +1263,14 @@ struct arm_smmu_attach_state {
struct arm_smmu_vmaster *vmaster;
struct arm_smmu_inv_state old_domain_invst;
struct arm_smmu_inv_state new_domain_invst;
struct arm_smmu_master_domain *old_master_domain;
bool ats_enabled;
};
int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
struct iommu_domain *new_domain);
void arm_smmu_attach_commit(struct arm_smmu_attach_state *state);
void arm_smmu_attach_release(struct arm_smmu_attach_state *state);
void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
const struct arm_smmu_ste *target);
drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+1 -1
View File
@@ -663,7 +663,7 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq)
/* Cap queue size to SMMU's IDR1.CMDQS and ensure natural alignment */
regval = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
q->llq.max_n_shift =
min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, regval));
arm_smmu_cmdq_max_n_shift(FIELD_GET(IDR1_CMDQS, regval));
/* Use the common helper to init the VCMDQ, and then... */
ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
drivers/iommu/bcm2712-iommu-cache.c
+84
View File
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* IOMMU driver for BCM2712 TLB cache
*
* Copyright (c) 2023 Raspberry Pi Ltd.
* Copyright (c) 2026 Daniel Drake
*
* The BCM2712 IOMMUC is a centralized TLB which accelerates address translation
* across the SoC's IOMMU devices. If an address mapping is not found in the
* IOMMU's local TLB cache, then this IOMMUC is consulted. The IOMMUC must be
* explicitly invalidated when modifying or unmapping IOMMU page tables.
*/
#include <linux/bits.h>
#include <linux/cleanup.h>
#include <linux/err.h>
#include <linux/iopoll.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include "bcm2712-iommu-cache.h"
struct bcm2712_iommu_cache {
struct device *dev;
spinlock_t hw_lock;
void __iomem *reg_base;
};
#define MMUC_CONTROL_ENABLE BIT(0)
#define MMUC_CONTROL_FLUSH BIT(1)
#define MMUC_CONTROL_FLUSHING BIT(2)
void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache)
{
u32 val;
int ret;
scoped_guard(spinlock_irqsave, &cache->hw_lock) {
writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH,
cache->reg_base);
ret = readl_poll_timeout_atomic(cache->reg_base, val,
!(val & MMUC_CONTROL_FLUSHING),
0, 50);
}
if (ret)
dev_err_ratelimited(cache->dev, "cache flush timed out\n");
}
static int bcm2712_iommu_cache_probe(struct platform_device *pdev)
{
struct bcm2712_iommu_cache *cache;
cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL);
if (!cache)
return -ENOMEM;
cache->dev = &pdev->dev;
spin_lock_init(&cache->hw_lock);
cache->reg_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(cache->reg_base))
return PTR_ERR(cache->reg_base);
platform_set_drvdata(pdev, cache);
return 0;
}
static const struct of_device_id bcm2712_iommu_cache_of_match[] = {
{ .compatible = "brcm,bcm2712-iommuc" },
{ /* sentinel */ },
};
static struct platform_driver bcm2712_iommu_cache_driver = {
.probe = bcm2712_iommu_cache_probe,
.driver = {
.name = "bcm2712-iommu-cache",
.of_match_table = bcm2712_iommu_cache_of_match,
.suppress_bind_attrs = true,
},
};
builtin_platform_driver(bcm2712_iommu_cache_driver);
drivers/iommu/bcm2712-iommu-cache.h
+9
View File
@@ -0,0 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _BCM2712_IOMMU_CACHE_H
#define _BCM2712_IOMMU_CACHE_H
struct bcm2712_iommu_cache;
void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache);
#endif
drivers/iommu/bcm2712-iommu.c
+550
View File
@@ -0,0 +1,550 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* IOMMU driver for Broadcom BCM2712
*
* Copyright (c) 2023-2025 Raspberry Pi Ltd.
* Copyright (c) 2026 Daniel Drake
*
* Each BCM2712 IOMMU has multiple devices hardwired into it, whose
* DMA transactions all route through the IOMMU. There is no stream ID tagging
* or any other kind of segmentation to differentiate between requests from
* different devices. It is also not possible to toggle a specific device
* between iommu-mapped and bypass modes.
*
* The hardware supports the simultaneous configuration of a bypass window
* (where memory accesses operate in identity mode) and a translation aperture
* (supporting page mapping). While combining these two would allow for
* GART-like operation, this driver's paging domain implementation configures
* only the translation aperture (starting at IOVA 0) offering a degree of
* memory protection.
*
* The page table format is a two-level format handled by generic_pt/bcm2712.
*/
#include <linux/bitfield.h>
#include <linux/cleanup.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/sizes.h>
#include <linux/generic_pt/iommu.h>
#include "bcm2712-iommu-cache.h"
#include "dma-iommu.h"
#include "iommu-pages.h"
/* BCM2712 IOMMU is organized around 4Kbyte pages */
#define IOMMU_PAGE_SHIFT 12
#define IOMMU_PAGE_SIZE (1ul << IOMMU_PAGE_SHIFT)
/* A PTE is 4 bytes */
#define PTE_SIZE_SHIFT 2
/* L1/L2 table sizing (IOMMU hardware pages): 1024 entries per page */
#define PTES_PER_IOPG_SHIFT (IOMMU_PAGE_SHIFT - PTE_SIZE_SHIFT)
#define MMMU_CTRL_OFFSET 0x00
#define MMMU_CTRL_CAP_EXCEEDED BIT(27)
#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN BIT(26)
#define MMMU_CTRL_CAP_EXCEEDED_INT_EN BIT(25)
#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN BIT(24)
#define MMMU_CTRL_PT_INVALID BIT(20)
#define MMMU_CTRL_PT_INVALID_ABORT_EN BIT(19)
#define MMMU_CTRL_PT_INVALID_INT_EN BIT(18)
#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN BIT(17)
#define MMMU_CTRL_PT_INVALID_EN BIT(16)
#define MMMU_CTRL_WRITE_VIOLATION BIT(12)
#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN BIT(11)
#define MMMU_CTRL_WRITE_VIOLATION_INT_EN BIT(10)
#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9)
#define MMMU_CTRL_BYPASS BIT(8)
#define MMMU_CTRL_TLB_CLEARING BIT(7)
#define MMMU_CTRL_STATS_CLEAR BIT(3)
#define MMMU_CTRL_TLB_CLEAR BIT(2)
#define MMMU_CTRL_STATS_ENABLE BIT(1)
#define MMMU_CTRL_ENABLE BIT(0)
#define MMMU_CTRL_OPERATING_FLAGS (\
MMMU_CTRL_CAP_EXCEEDED_ABORT_EN | \
MMMU_CTRL_PT_INVALID_ABORT_EN | \
MMMU_CTRL_PT_INVALID_EN | \
MMMU_CTRL_WRITE_VIOLATION_ABORT_EN | \
MMMU_CTRL_STATS_ENABLE | \
MMMU_CTRL_ENABLE)
#define MMMU_PT_PA_BASE_OFFSET 0x04
#define MMMU_ADDR_CAP_OFFSET 0x14
#define MMMU_ADDR_CAP_ENABLE BIT(31)
#define ADDR_CAP_SHIFT ilog2(SZ_256M)
#define MMMU_SHOOT_DOWN_OFFSET 0x18
#define MMMU_SHOOT_DOWN_SHOOTING BIT(31)
#define MMMU_SHOOT_DOWN_SHOOT BIT(30)
#define MMMU_BYPASS_START_OFFSET 0x1c
#define MMMU_BYPASS_START_ENABLE BIT(31)
#define MMMU_BYPASS_END_OFFSET 0x20
#define MMMU_BYPASS_END_ENABLE BIT(31)
#define MMMU_MISC_OFFSET 0x24
#define MMMU_MISC_SINGLE_TABLE BIT(31)
#define MMMU_ILLEGAL_ADR_OFFSET 0x30
#define MMMU_ILLEGAL_ADR_ENABLE BIT(31)
#define MMMU_DEBUG_INFO_OFFSET 0x38
#define MMMU_DEBUG_INFO_VERSION_MASK 0x0000000Fu
#define MMMU_DEBUG_INFO_VA_WIDTH_MASK 0x000000F0u
#define MMMU_DEBUG_INFO_PA_WIDTH_MASK 0x00000F00u
#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK 0x000FF000u
#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK 0x0FF00000u
#define MMMU_DEBUG_INFO_BYPASS_4M BIT(28)
#define MMMU_DEBUG_INFO_BYPASS BIT(29)
struct bcm2712_iommu {
struct device *dev;
struct iommu_device iommu;
struct bcm2712_iommu_domain *domain;
struct bcm2712_iommu_cache *cache;
void __iomem *reg_base;
spinlock_t hw_lock;
size_t bigpage_size;
size_t superpage_size;
};
struct bcm2712_iommu_domain {
union {
struct iommu_domain base;
struct pt_iommu_bcm2712 pt;
};
struct bcm2712_iommu *mmu;
void *default_page;
};
static inline void bcm2712_iommu_writel(struct bcm2712_iommu *mmu,
unsigned int offset, u32 val)
{
writel(val, mmu->reg_base + offset);
}
static inline u32 bcm2712_iommu_readl(struct bcm2712_iommu *mmu,
unsigned int offset)
{
return readl(mmu->reg_base + offset);
}
static struct bcm2712_iommu_domain *
to_bcm2712_domain(struct iommu_domain *domain)
{
return container_of(domain, struct bcm2712_iommu_domain, base);
}
static void bcm2712_iommu_init(struct bcm2712_iommu *mmu)
{
unsigned int bigpage_width, superpage_width;
u32 u = bcm2712_iommu_readl(mmu, MMMU_DEBUG_INFO_OFFSET);
u32 pa_width = FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u);
dev_dbg(mmu->dev, "DEBUG_INFO = 0x%08x\n", u);
WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 ||
FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 ||
pa_width < 6 || !(u & MMMU_DEBUG_INFO_BYPASS));
dma_set_mask_and_coherent(mmu->dev, DMA_BIT_MASK(pa_width + 30u));
bigpage_width = FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u);
if (bigpage_width)
mmu->bigpage_size = IOMMU_PAGE_SIZE << bigpage_width;
superpage_width = FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u);
if (superpage_width)
mmu->superpage_size = IOMMU_PAGE_SIZE << superpage_width;
/* Disable MMU and clear sticky flags */
bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
MMMU_CTRL_CAP_EXCEEDED | MMMU_CTRL_PT_INVALID |
MMMU_CTRL_WRITE_VIOLATION |
MMMU_CTRL_STATS_CLEAR);
/* Put MMU into 2-level mode */
bcm2712_iommu_writel(mmu, MMMU_MISC_OFFSET,
bcm2712_iommu_readl(mmu, MMMU_MISC_OFFSET) &
~MMMU_MISC_SINGLE_TABLE);
}
static int bcm2712_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev,
struct iommu_domain *old)
{
struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
guard(spinlock_irqsave)(&mmu->hw_lock);
bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
mmu->domain = NULL;
return 0;
}
static struct iommu_domain bcm2712_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &(const struct iommu_domain_ops) {
.attach_dev = bcm2712_iommu_identity_attach,
},
};
static int bcm2712_iommu_clear_and_enable(struct bcm2712_iommu *mmu)
__must_hold(&mmu->hw_lock)
{
u32 val;
bcm2712_iommu_cache_flush(mmu->cache);
bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
MMMU_CTRL_OPERATING_FLAGS | MMMU_CTRL_TLB_CLEAR);
return readl_poll_timeout_atomic(mmu->reg_base + MMMU_CTRL_OFFSET, val,
!(val & MMMU_CTRL_TLB_CLEARING), 0,
50);
}
static int bcm2712_iommu_attach_dev(struct iommu_domain *domain,
struct device *dev,
struct iommu_domain *old)
{
struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
struct pt_iommu_bcm2712_hw_info info;
u32 default_page_pfn;
int ret = 0;
/* all attached devices must belong to the same IOMMU instance */
if (mydomain->mmu != mmu)
return -EINVAL;
scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
if (mmu->domain == mydomain)
break;
/* Configure translation aperture */
bcm2712_iommu_writel(mmu, MMMU_ADDR_CAP_OFFSET,
MMMU_ADDR_CAP_ENABLE |
(domain->geometry.aperture_end >>
ADDR_CAP_SHIFT));
/*
* When the IOMMU handles a request, it adds the
* PT_PA_BASE_OFFSET to (IOVA>>32) to calculate the PFN of the
* corresponding L1 directory page.
* IOVA bits [31:22] are then used to fetch the L1 descriptor
* within (which in turn points to the L2 table).
* This clever logic would allow for a L1 table larger than 4kb
* (and hence a larger aperture).
*/
pt_iommu_bcm2712_hw_info(&mydomain->pt, &info);
bcm2712_iommu_writel(mmu, MMMU_PT_PA_BASE_OFFSET,
info.pt_base >> IOMMU_PAGE_SHIFT);
/* Disable bypass window */
bcm2712_iommu_writel(mmu, MMMU_BYPASS_START_OFFSET, 0);
bcm2712_iommu_writel(mmu, MMMU_BYPASS_END_OFFSET, 0);
/*
* A default (error) page is used to catch illegal reads/writes.
*/
default_page_pfn = virt_to_phys(mydomain->default_page) >>
IOMMU_PAGE_SHIFT;
bcm2712_iommu_writel(mmu, MMMU_ILLEGAL_ADR_OFFSET,
MMMU_ILLEGAL_ADR_ENABLE |
default_page_pfn);
ret = bcm2712_iommu_clear_and_enable(mmu);
if (ret) {
bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
mmu->domain = NULL;
break;
}
mmu->domain = mydomain;
}
if (ret)
dev_err_ratelimited(mmu->dev,
"TLB clear timed out during attach\n");
return ret;
}
static int bcm2712_iommu_shootdown_range(struct bcm2712_iommu *mmu,
unsigned long iova, size_t size)
__must_hold(&mmu->hw_lock)
{
unsigned long start_pfn = iova >> IOMMU_PAGE_SHIFT;
unsigned long end_pfn = (iova + size - 1) >> IOMMU_PAGE_SHIFT;
unsigned long pfn;
u32 val;
int ret;
/* Shootdown register deals with 4 pages at a time */
for (pfn = round_down(start_pfn, 4); pfn <= end_pfn; pfn += 4) {
bcm2712_iommu_writel(mmu, MMMU_SHOOT_DOWN_OFFSET,
MMMU_SHOOT_DOWN_SHOOT | pfn);
ret = readl_poll_timeout_atomic(
mmu->reg_base + MMMU_SHOOT_DOWN_OFFSET, val,
!(val & MMMU_SHOOT_DOWN_SHOOTING), 0, 50);
if (ret)
return ret;
}
return 0;
}
static int bcm2712_iommu_sync_range(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
struct bcm2712_iommu *mmu = mydomain->mmu;
int ret = 0;
if (!size)
return 0;
scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
if (mmu->domain != mydomain)
return 0;
/* If invalidating more than 1MB, just do a full TLB clear */
if (size >= SZ_1M) {
ret = bcm2712_iommu_clear_and_enable(mmu);
} else {
bcm2712_iommu_cache_flush(mmu->cache);
ret = bcm2712_iommu_shootdown_range(mmu, iova, size);
}
}
if (ret)
dev_err_ratelimited(mmu->dev,
"TLB sync timed out (size=%#zx)\n", size);
return ret;
}
static void bcm2712_iommu_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
bcm2712_iommu_sync_range(domain, gather->start,
gather->end - gather->start + 1);
iommu_put_pages_list(&gather->freelist);
}
static int bcm2712_iommu_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
return bcm2712_iommu_sync_range(domain, iova, size);
}
static void bcm2712_iommu_sync_all(struct iommu_domain *domain)
{
size_t aperture_size = domain->geometry.aperture_end -
domain->geometry.aperture_start + 1;
bcm2712_iommu_sync_range(domain, domain->geometry.aperture_start,
aperture_size);
}
static void bcm2712_iommu_domain_free(struct iommu_domain *domain)
{
struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
struct bcm2712_iommu *mmu = mydomain->mmu;
scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
if (mmu->domain == mydomain) {
bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
mmu->domain = NULL;
}
}
pt_iommu_deinit(&mydomain->pt.iommu);
if (mydomain->default_page)
iommu_free_pages(mydomain->default_page);
kfree(mydomain);
}
static const struct iommu_domain_ops bcm2712_paging_domain_ops = {
IOMMU_PT_DOMAIN_OPS(bcm2712),
.attach_dev = bcm2712_iommu_attach_dev,
.iotlb_sync = bcm2712_iommu_sync,
.iotlb_sync_map = bcm2712_iommu_sync_map,
.flush_iotlb_all = bcm2712_iommu_sync_all,
.free = bcm2712_iommu_domain_free,
};
static struct iommu_domain *bcm2712_iommu_domain_alloc(struct device *dev)
{
struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
struct bcm2712_iommu_domain *domain;
struct pt_iommu_bcm2712_cfg cfg = {};
int ret;
domain = kzalloc_obj(*domain);
if (!domain)
return NULL;
domain->mmu = mmu;
domain->pt.iommu.iommu_device = mmu->dev;
cfg.common.features = BIT(PT_FEAT_DMA_INCOHERENT);
/* Bigpage and superpage sizes are typically 64K and 1M, but may vary */
if (mmu->bigpage_size)
cfg.bigpage_lg2 = ilog2(mmu->bigpage_size);
if (mmu->superpage_size)
cfg.superpage_lg2 = ilog2(mmu->superpage_size);
/* 2-level format: 10-bit L1 + 10-bit L2 + 12-bit page offset */
cfg.common.hw_max_vasz_lg2 =
(2 * PTES_PER_IOPG_SHIFT) + IOMMU_PAGE_SHIFT;
/* PTEs encode a 25-bit output address PFN */
cfg.common.hw_max_oasz_lg2 = 25 + IOMMU_PAGE_SHIFT;
ret = pt_iommu_bcm2712_init(&domain->pt, &cfg, GFP_KERNEL);
if (ret)
goto err;
/* Set up a default (error) page used to catch illegal reads/writes */
domain->default_page = iommu_alloc_pages_sz(GFP_KERNEL, PAGE_SIZE);
if (!domain->default_page)
goto err;
domain->base.ops = &bcm2712_paging_domain_ops;
return &domain->base;
err:
bcm2712_iommu_domain_free(&domain->base);
return NULL;
}
static struct bcm2712_iommu *
bcm2712_iommu_get_by_fwnode(struct fwnode_handle *fwnode)
{
struct device *dev __free(put_device) =
bus_find_device_by_fwnode(&platform_bus_type, fwnode);
return dev ? dev_get_drvdata(dev) : NULL;
}
static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct bcm2712_iommu *mmu;
if (!fwspec || !fwspec->iommu_fwnode)
return ERR_PTR(-ENODEV);
mmu = bcm2712_iommu_get_by_fwnode(fwspec->iommu_fwnode);
if (!mmu)
return ERR_PTR(-ENODEV);
dev_iommu_priv_set(dev, mmu);
return &mmu->iommu;
}
static int bcm2712_iommu_of_xlate(struct device *dev,
const struct of_phandle_args *args)
{
return iommu_fwspec_add_ids(dev, args->args, 0);
}
static const struct iommu_ops bcm2712_iommu_ops = {
.identity_domain = &bcm2712_identity_domain,
.domain_alloc_paging = bcm2712_iommu_domain_alloc,
.probe_device = bcm2712_iommu_probe_device,
.device_group = generic_single_device_group,
.get_resv_regions = iommu_dma_get_resv_regions,
.of_xlate = bcm2712_iommu_of_xlate,
};
static const struct of_device_id bcm2712_iommu_of_match[] = {
{ .compatible = "brcm,bcm2712-iommu" },
{ /* sentinel */ }
};
static int bcm2712_iommu_init_cache(struct bcm2712_iommu *mmu,
struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct device_node *cache_np __free(device_node) =
of_parse_phandle(dev->of_node, "brcm,iommu-cache", 0);
if (!cache_np)
return dev_err_probe(dev, -ENOENT,
"missing brcm,iommu-cache property\n");
struct platform_device *cache_pdev __free(platform_device_put) =
of_find_device_by_node(cache_np);
if (!cache_pdev)
return dev_err_probe(dev, -EPROBE_DEFER,
"waiting for cache device\n");
mmu->cache = platform_get_drvdata(cache_pdev);
if (!mmu->cache)
return dev_err_probe(dev, -EPROBE_DEFER,
"waiting for cache driver probe\n");
return 0;
}
static int bcm2712_iommu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct bcm2712_iommu *mmu;
int ret;
mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
if (!mmu)
return -ENOMEM;
mmu->dev = dev;
spin_lock_init(&mmu->hw_lock);
mmu->reg_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(mmu->reg_base))
return PTR_ERR(mmu->reg_base);
ret = bcm2712_iommu_init_cache(mmu, pdev);
if (ret)
return ret;
platform_set_drvdata(pdev, mmu);
bcm2712_iommu_init(mmu);
ret = iommu_device_sysfs_add(&mmu->iommu, dev, NULL, "%s",
dev_name(dev));
if (ret)
return ret;
ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, dev);
if (ret) {
iommu_device_sysfs_remove(&mmu->iommu);
return ret;
}
return 0;
}
static struct platform_driver bcm2712_iommu_driver = {
.driver = {
.name = "bcm2712-iommu",
.of_match_table = bcm2712_iommu_of_match,
.suppress_bind_attrs = true,
},
.probe = bcm2712_iommu_probe,
};
builtin_platform_driver(bcm2712_iommu_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Daniel Drake <dan@reactivated.net>");
MODULE_DESCRIPTION("Broadcom BCM2712 IOMMU driver");
drivers/iommu/dma-iommu.c
+20 -12
View File
@@ -37,6 +37,7 @@
#include "dma-iommu.h"
#include "iommu-pages.h"
#include "iommu-priv.h"
struct iommu_dma_msi_page {
struct list_head list;
@@ -1477,6 +1478,11 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
sg_dma_len(s) = s_length;
s->offset -= s_iova_off;
s_length = iova_align(iovad, s_length + s_iova_off);
if (overflows_type(s_length, s->length)) {
ret = -EOVERFLOW;
goto out_restore_sg;
}
s->length = s_length;
/*
@@ -1493,7 +1499,20 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
* time through here (i.e. before it has a meaningful value).
*/
if (pad_len && pad_len < s_length - 1) {
prev->length += pad_len;
unsigned int new_prev_len;
/*
* For large mappings spanning multiple GBs we
* may not be able to fit all needed padding into
* sg->length.
*/
if (check_add_overflow(prev->length, pad_len, &new_prev_len)) {
ret = -EOVERFLOW;
goto out_restore_sg;
}
prev->length = new_prev_len;
iova_len += pad_len;
}
@@ -2163,12 +2182,6 @@ out_err:
dev_clear_dma_iommu(dev);
}
static bool has_msi_cookie(const struct iommu_domain *domain)
{
return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA ||
domain->cookie_type == IOMMU_COOKIE_DMA_MSI);
}
static size_t cookie_msi_granule(const struct iommu_domain *domain)
{
switch (domain->cookie_type) {
@@ -2250,11 +2263,6 @@ int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
struct device *dev = msi_desc_to_dev(desc);
const struct iommu_dma_msi_page *msi_page;
if (!has_msi_cookie(domain)) {
msi_desc_set_iommu_msi_iova(desc, 0, 0);
return 0;
}
iommu_group_mutex_assert(dev);
msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
if (!msi_page)
drivers/iommu/exynos-iommu.c
+33 -4
View File
@@ -8,6 +8,7 @@
#define DEBUG
#endif
#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
@@ -143,6 +144,10 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
#define CTRL_VM_FAULT_MODE_STALL BIT(3)
#define CAPA0_CAPA1_EXIST BIT(11)
#define CAPA1_VCR_ENABLED BIT(14)
#define CAPA1_NO_BLOCK_MODE BIT(15)
#define FAULT_INFO_AXID GENMASK(15, 0)
#define FAULT_INFO_AXLEN GENMASK(19, 16)
#define FAULT_INFO_WRITE BIT(20)
/* common registers */
#define REG_MMU_CTRL 0x000
@@ -193,6 +198,7 @@ struct sysmmu_fault {
sysmmu_iova_t addr; /* IOVA address that caused fault */
const char *name; /* human readable fault name */
unsigned int type; /* fault type for report_iommu_fault() */
u32 info; /* raw transaction info (v7+ only) */
};
struct sysmmu_v1_fault_info {
@@ -306,6 +312,7 @@ struct sysmmu_drvdata {
/* v7 fields */
bool has_vcr; /* virtual machine control register */
bool no_block; /* BLOCK mode not implemented */
};
#define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg)
@@ -358,7 +365,8 @@ static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data,
fault->addr = readl(SYSMMU_REG(data, fault_va));
fault->name = sysmmu_v7_fault_names[itype % 4];
fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
fault->type = (info & FAULT_INFO_WRITE) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
fault->info = info;
return 0;
}
@@ -511,6 +519,7 @@ static void __sysmmu_get_vcr(struct sysmmu_drvdata *data)
u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1);
data->has_vcr = capa1 & CAPA1_VCR_ENABLED;
data->no_block = capa1 & CAPA1_NO_BLOCK_MODE;
}
static void __sysmmu_get_version(struct sysmmu_drvdata *data)
@@ -555,6 +564,12 @@ static void show_fault_information(struct sysmmu_drvdata *data,
dev_name(data->master),
fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE",
fault->name, fault->addr);
/* AxID identifies the issuing port inside the master */
if (data->variant->fault_info)
dev_err(data->sysmmu, "transaction info %#010x: AxID %#lx, AxLEN %lu\n",
fault->info,
FIELD_GET(FAULT_INFO_AXID, fault->info),
FIELD_GET(FAULT_INFO_AXLEN, fault->info));
dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable);
ent = section_entry(phys_to_virt(data->pgtable), fault->addr);
dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent);
@@ -652,7 +667,12 @@ static void __sysmmu_enable(struct sysmmu_drvdata *data)
__sysmmu_enable_clocks(data);
spin_lock_irqsave(&data->lock, flags);
writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
/*
* On no-block hardware CTRL_BLOCK acts as a plain enable; keep the
* MMU disabled until it is fully programmed.
*/
if (!data->no_block)
writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
__sysmmu_init_config(data);
__sysmmu_set_ptbase(data, data->pgtable);
__sysmmu_enable_vid(data);
@@ -677,7 +697,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
spin_lock_irqsave(&data->lock, flags);
if (data->active && data->version >= MAKE_MMU_VER(3, 3)) {
clk_enable(data->clk_master);
if (sysmmu_block(data)) {
/*
* No-block hardware accepts invalidation writes while the
* MMU is running; it is also v7+, so flush-all covers the
* FLPD cache.
*/
if (data->no_block) {
__sysmmu_tlb_invalidate(data);
} else if (sysmmu_block(data)) {
if (data->version >= MAKE_MMU_VER(5, 0))
__sysmmu_tlb_invalidate(data);
else
@@ -713,7 +740,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
if (MMU_MAJ_VER(data->version) == 2)
num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64);
if (sysmmu_block(data)) {
if (data->no_block) {
__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
} else if (sysmmu_block(data)) {
__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
sysmmu_unblock(data);
}
drivers/iommu/generic_pt/.kunitconfig
+1
View File
@@ -7,6 +7,7 @@ CONFIG_IOMMU_PT_AMDV1=y
CONFIG_IOMMU_PT_VTDSS=y
CONFIG_IOMMU_PT_RISCV64=y
CONFIG_IOMMU_PT_X86_64=y
CONFIG_IOMMU_PT_BCM2712=y
CONFIG_IOMMU_PT_KUNIT_TEST=y
CONFIG_IOMMUFD=y
drivers/iommu/generic_pt/Kconfig
+10
View File
@@ -72,6 +72,15 @@ config IOMMU_PT_X86_64
Selected automatically by an IOMMU driver that uses this format.
config IOMMU_PT_BCM2712
tristate "IOMMU page table for Broadcom BCM2712"
help
iommu_domain implementation for the Broadcom BCM2712 IOMMU found on
Raspberry Pi 5. It supports 4K page sizes over a 2-level page table
format.
Selected automatically by an IOMMU driver that uses this format.
config IOMMU_PT_KUNIT_TEST
tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS
depends on KUNIT
@@ -79,6 +88,7 @@ config IOMMU_PT_KUNIT_TEST
depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64
depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64
depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS
depends on IOMMU_PT_BCM2712 || !IOMMU_PT_BCM2712
default KUNIT_ALL_TESTS
help
Enable kunit tests for GENERIC_PT and IOMMU_PT that covers all the
drivers/iommu/generic_pt/fmt/Makefile
+2
View File
@@ -9,6 +9,8 @@ iommu_pt_fmt-$(CONFIG_IOMMU_PT_RISCV64) += riscv64
iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64
iommu_pt_fmt-$(CONFIG_IOMMU_PT_BCM2712) += bcm2712
IOMMU_PT_KUNIT_TEST :=
define create_format
obj-$(2) += iommu_$(1).o
drivers/iommu/generic_pt/fmt/bcm2712.h
+288
View File
@@ -0,0 +1,288 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2026 Daniel Drake
*
* BCM2712 IOMMU simple two level page table
*/
#ifndef __GENERIC_PT_FMT_BCM2712_H
#define __GENERIC_PT_FMT_BCM2712_H
#include "defs_bcm2712.h"
#include "../pt_defs.h"
#include <linux/bitfield.h>
#include <linux/container_of.h>
#include <linux/log2.h>
#include <linux/sizes.h>
enum {
/* Hardware provides a two-level page table */
PT_MAX_TOP_LEVEL = 1,
/* Hardware page size is strictly 4kb */
PT_GRANULE_LG2SZ = ilog2(SZ_4K),
PT_TABLEMEM_LG2SZ = PT_GRANULE_LG2SZ,
/* Table entries and leaf entries are 32 bits */
PT_ITEM_WORD_SIZE = sizeof(pt_bcm2712_entry_t),
/* Leaf entries encode a 25-bit PFN */
PT_MAX_OUTPUT_ADDRESS_LG2 = 25 + PT_GRANULE_LG2SZ,
/*
* For simplicity, only manage mappings within an address space of
* exactly 4GB. This is because our Level 1 directory page will be 4kb
* in the smallest case, permitting 1024 entries pointing at Level 0
* pages, each permitting 4mb of mapped memory.
*/
PT_MAX_VA_ADDRESS_LG2 = ilog2(SZ_4G),
/* Level 1 base address is programmed as a 32-bit PFN */
PT_TOP_PHYS_MASK = GENMASK_ULL(31 + PT_GRANULE_LG2SZ, PT_GRANULE_LG2SZ),
};
/* PTE bits */
enum {
BCM2712PT_VALID = BIT(28),
BCM2712PT_WRITE = BIT(29),
BCM2712PT_PAGESIZE = GENMASK(31, 30),
BCM2712PT_PFN = GENMASK(24, 0),
};
#define common_to_bcm2712pt(common_ptr) \
container_of_const(common_ptr, struct pt_bcm2712, common)
#define to_bcm2712pt(pts) common_to_bcm2712pt((pts)->range->common)
static inline pt_oaddr_t bcm2712pt_table_pa(const struct pt_state *pts)
{
return oalog2_mul(FIELD_GET(BCM2712PT_PFN, pts->entry),
PT_GRANULE_LG2SZ);
}
#define pt_table_pa bcm2712pt_table_pa
#define pt_item_oa bcm2712pt_table_pa
static inline bool bcm2712pt_can_have_leaf(const struct pt_state *pts)
{
return true;
}
#define pt_can_have_leaf bcm2712pt_can_have_leaf
/* 4MB pages are installed at level 1, everything else at level 0 */
static inline unsigned int bcm2712pt_pgsz_lg2_to_level(struct pt_common *common,
unsigned int pgsize_lg2)
{
return pgsize_lg2 == ilog2(SZ_4M) ? 1 : 0;
}
#define pt_pgsz_lg2_to_level bcm2712pt_pgsz_lg2_to_level
static inline pt_vaddr_t bcm2712pt_possible_sizes(const struct pt_state *pts)
{
struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common);
if (pts->level == 1)
return SZ_4M;
return SZ_4K |
(table->bigpage_lg2 ? BIT_ULL(table->bigpage_lg2) : 0) |
(table->superpage_lg2 ? BIT_ULL(table->superpage_lg2) : 0);
}
#define pt_possible_sizes bcm2712pt_possible_sizes
static inline unsigned int
bcm2712pt_entry_num_contig_lg2(const struct pt_state *pts)
{
struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common);
u32 pgsz = FIELD_GET(BCM2712PT_PAGESIZE, pts->entry);
if (pts->level != 0 || !(pts->entry & BCM2712PT_VALID))
return 0;
/*
* Superpage/bigpage contiguous mapping (really just hinting) is handled
* via PAGESIZE bits on each leaf entry.
*/
if (pgsz == 2 && table->superpage_lg2)
return table->superpage_lg2 - PT_GRANULE_LG2SZ;
else if (pgsz == 1 && table->bigpage_lg2)
return table->bigpage_lg2 - PT_GRANULE_LG2SZ;
return 0;
}
#define pt_entry_num_contig_lg2 bcm2712pt_entry_num_contig_lg2
static inline unsigned int bcm2712pt_num_items_lg2(const struct pt_state *pts)
{
return PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE);
}
#define pt_num_items_lg2 bcm2712pt_num_items_lg2
static inline enum pt_entry_type bcm2712pt_load_entry_raw(struct pt_state *pts)
{
const pt_bcm2712_entry_t *tablep =
pt_cur_table(pts, pt_bcm2712_entry_t);
pts->entry = READ_ONCE(tablep[pts->index]);
if (!(pts->entry & BCM2712PT_VALID))
return PT_ENTRY_EMPTY;
if (pts->level == 1) {
if (FIELD_GET(BCM2712PT_PAGESIZE, pts->entry) == 3)
return PT_ENTRY_OA;
return PT_ENTRY_TABLE;
}
return PT_ENTRY_OA;
}
#define pt_load_entry_raw bcm2712pt_load_entry_raw
static inline void
bcm2712pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
unsigned int oasz_lg2,
const struct pt_write_attrs *attrs)
{
pt_bcm2712_entry_t *tablep = pt_cur_table(pts, pt_bcm2712_entry_t);
pt_bcm2712_entry_t entry;
if (!pt_check_install_leaf_args(pts, oa, oasz_lg2))
return;
entry = BCM2712PT_VALID | attrs->descriptor_bits |
FIELD_PREP(BCM2712PT_PFN, oalog2_div(oa, PT_GRANULE_LG2SZ));
if (pts->level == 1) {
/* Level 1 hugepage (4MB) */
entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 3);
WRITE_ONCE(tablep[pts->index], entry);
pts->entry = entry;
return;
}
if (oasz_lg2 == PT_GRANULE_LG2SZ) {
WRITE_ONCE(tablep[pts->index], entry);
pts->entry = entry;
} else {
struct pt_bcm2712 *table =
common_to_bcm2712pt(pts->range->common);
u32 *end;
tablep += pts->index;
end = tablep + log2_to_int(oasz_lg2 - PT_GRANULE_LG2SZ);
/*
* Leaf entries can contain hints indicating bigpage/superpage
* contiguous mappings to permit TLB optimization
*/
if (oasz_lg2 == table->superpage_lg2)
entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 2);
else if (oasz_lg2 == table->bigpage_lg2)
entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 1);
pts->entry = entry;
for (; tablep != end; tablep++, entry++)
WRITE_ONCE(*tablep, entry);
}
}
#define pt_install_leaf_entry bcm2712pt_install_leaf_entry
static inline bool bcm2712pt_install_table(struct pt_state *pts,
pt_oaddr_t table_pa,
const struct pt_write_attrs *attrs)
{
pt_bcm2712_entry_t entry =
BCM2712PT_VALID |
FIELD_PREP(BCM2712PT_PFN,
oalog2_div(table_pa, PT_GRANULE_LG2SZ));
return pt_table_install32(pts, entry);
}
#define pt_install_table bcm2712pt_install_table
static inline void bcm2712pt_attr_from_entry(const struct pt_state *pts,
struct pt_write_attrs *attrs)
{
attrs->descriptor_bits = pts->entry & BCM2712PT_WRITE;
}
#define pt_attr_from_entry bcm2712pt_attr_from_entry
static inline unsigned int bcm2712pt_max_sw_bit(struct pt_common *common)
{
return 2;
}
#define pt_max_sw_bit bcm2712pt_max_sw_bit
static inline u32 bcm2712pt_sw_bit(unsigned int bitnr)
{
if (__builtin_constant_p(bitnr) && bitnr > 2)
BUILD_BUG();
if (PT_WARN_ON(bitnr > 2))
return 0;
/* Bits 27, 26 and 25 are believed spare */
return BIT(27 - bitnr);
}
#define pt_sw_bit bcm2712pt_sw_bit
/* --- iommu */
#include <linux/generic_pt/iommu.h>
#include <linux/iommu.h>
#define pt_iommu_table pt_iommu_bcm2712
/* The common struct is in the per-format common struct */
static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table)
{
return &container_of(iommu_table, struct pt_iommu_table, iommu)
->bcm2712pt.common;
}
static inline struct pt_iommu *iommu_from_common(struct pt_common *common)
{
return &container_of(common, struct pt_iommu_table, bcm2712pt.common)
->iommu;
}
static inline int bcm2712_pt_iommu_set_prot(struct pt_common *common,
struct pt_write_attrs *attrs,
unsigned int iommu_prot)
{
attrs->descriptor_bits = 0;
if (iommu_prot & IOMMU_WRITE)
attrs->descriptor_bits |= BCM2712PT_WRITE;
return 0;
}
#define pt_iommu_set_prot bcm2712_pt_iommu_set_prot
static inline int bcm2712_pt_fmt_init(struct pt_iommu_table *fmt_table,
const struct pt_iommu_bcm2712_cfg *cfg)
{
fmt_table->bcm2712pt.bigpage_lg2 = cfg->bigpage_lg2;
fmt_table->bcm2712pt.superpage_lg2 = cfg->superpage_lg2;
pt_top_set_level(&fmt_table->bcm2712pt.common, PT_MAX_TOP_LEVEL);
return 0;
}
#define pt_iommu_fmt_init bcm2712_pt_fmt_init
static inline void
bcm2712pt_iommu_fmt_hw_info(struct pt_iommu_bcm2712 *table,
const struct pt_range *top_range,
struct pt_iommu_bcm2712_hw_info *info)
{
info->pt_base = virt_to_phys(top_range->top_table);
PT_WARN_ON(info->pt_base & ~PT_TOP_PHYS_MASK);
}
#define pt_iommu_fmt_hw_info bcm2712pt_iommu_fmt_hw_info
#if defined(GENERIC_PT_KUNIT)
static const struct pt_iommu_bcm2712_cfg bcm2712_kunit_fmt_cfgs[] = {
[0] = {
.common.hw_max_vasz_lg2 = 32,
.common.hw_max_oasz_lg2 = 37,
.bigpage_lg2 = 16,
.superpage_lg2 = 20,
},
};
#define kunit_fmt_cfgs bcm2712_kunit_fmt_cfgs
enum { KUNIT_FMT_FEATURES = 0 };
#endif
#endif
drivers/iommu/generic_pt/fmt/defs_bcm2712.h
+18
View File
@@ -0,0 +1,18 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __GENERIC_PT_FMT_DEFS_BCM2712_H
#define __GENERIC_PT_FMT_DEFS_BCM2712_H
#include <linux/generic_pt/common.h>
#include <linux/types.h>
typedef u32 pt_bcm2712_entry_t;
typedef u64 pt_vaddr_t;
typedef u64 pt_oaddr_t;
struct bcm2712pt_write_attrs {
pt_bcm2712_entry_t descriptor_bits;
gfp_t gfp;
};
#define pt_write_attrs bcm2712pt_write_attrs
#endif
drivers/iommu/generic_pt/fmt/iommu_bcm2712.c
+6
View File
@@ -0,0 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
#define PT_FMT bcm2712
#define PT_SUPPORTED_FEATURES BIT(PT_FEAT_DMA_INCOHERENT)
#include <linux/generic_pt/iommu.h>
#include "iommu_template.h"
drivers/iommu/generic_pt/kunit_iommu_pt.h
+1 -1
View File
@@ -403,7 +403,7 @@ static void test_pgsize_boundary(struct kunit *test)
priv->smallest_pgsz != SZ_4K)
kunit_skip(test, "Format does not have the required range");
do_map(test, 0xfef80000, 0x208b95d000, 0xfef9ffff - 0xfef80000 + 1);
do_map(test, 0xfef80000, 0x8b95d000, 0xfef9ffff - 0xfef80000 + 1);
}
/* See https://lore.kernel.org/r/20250826143816.38686-1-eugkoira@amazon.com */
drivers/iommu/hyperv/Kconfig
+15
View File
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-only
# Hyper-V para-virtualized IOMMU driver for Linux guests
config HYPERV_PVIOMMU
bool "Microsoft Hyper-V para-virtualized IOMMU driver for Linux guests"
depends on X86_64 && HYPERV && PCI
select IOMMU_DMA
select GENERIC_PT
select IOMMU_PT
select IOMMU_PT_X86_64
default HYPERV
help
Para-virtualized IOMMU driver for Linux guests running on
Microsoft Hyper-V. Provides DMA remapping and IOTLB
flush support to enable DMA isolation for PCI devices
assigned to the guest.
drivers/iommu/hyperv/Makefile
+1
View File
@@ -1,2 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_IRQ_REMAP) += hv-irq-remap-x86.o
obj-$(CONFIG_HYPERV_PVIOMMU) += hv-iommu-guest.o
drivers/iommu/hyperv/hv-iommu-guest.c
+682
View File
@@ -0,0 +1,682 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Hyper-V para-virtualized IOMMU driver for Linux guests.
*
* Copyright (C) 2024-2026 Microsoft, Inc.
*/
#define pr_fmt(fmt) "Hyper-V pvIOMMU: " fmt
#define dev_fmt(fmt) pr_fmt(fmt)
#include <linux/iommu.h>
#include <linux/pci.h>
#include <linux/dma-map-ops.h>
#include <linux/generic_pt/iommu.h>
#include <linux/pci-ats.h>
#include <asm/iommu.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
#include "iommu.h"
#include "../iommu-pages.h"
static struct hv_iommu_dev *hv_iommu_device;
/*
* Identity and blocking domains are static singletons: identity is a 1:1
* passthrough with no page table, blocking rejects all DMA. Neither holds
* per-IOMMU state, so one instance suffices even with multiple vIOMMUs.
*/
static const struct iommu_domain_ops hv_iommu_identity_domain_ops;
static const struct iommu_domain_ops hv_iommu_blocking_domain_ops;
static struct iommu_ops hv_iommu_ops;
static struct hv_iommu_domain hv_identity_domain = {
.domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &hv_iommu_identity_domain_ops,
.owner = &hv_iommu_ops,
},
};
static struct hv_iommu_domain hv_blocking_domain = {
.domain = {
.type = IOMMU_DOMAIN_BLOCKED,
.ops = &hv_iommu_blocking_domain_ops,
.owner = &hv_iommu_ops,
},
};
static inline bool hv_iommu_present(u64 cap)
{
return cap & HV_IOMMU_CAP_PRESENT;
}
static inline bool hv_iommu_s1_domain_supported(u64 cap)
{
return cap & HV_IOMMU_CAP_S1;
}
static inline bool hv_iommu_5lvl_supported(u64 cap)
{
return cap & HV_IOMMU_CAP_S1_5LVL;
}
static inline bool hv_iommu_ats_supported(u64 cap)
{
return cap & HV_IOMMU_CAP_ATS;
}
static int hv_create_device_domain(struct hv_iommu_domain *hv_domain)
{
int ret;
u64 status;
unsigned long flags;
struct hv_input_create_device_domain *input;
ret = ida_alloc_range(&hv_iommu_device->domain_ids,
hv_iommu_device->first_domain,
hv_iommu_device->last_domain, GFP_KERNEL);
if (ret < 0)
return ret;
hv_domain->device_domain.partition_id = HV_PARTITION_ID_SELF;
hv_domain->device_domain.domain_id.type = HV_DEVICE_DOMAIN_TYPE_S1;
hv_domain->device_domain.domain_id.id = ret;
hv_domain->hv_iommu = hv_iommu_device;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->device_domain = hv_domain->device_domain;
input->create_device_domain_flags.forward_progress_required = 1;
input->create_device_domain_flags.inherit_owning_vtl = 0;
status = hv_do_hypercall(HVCALL_CREATE_DEVICE_DOMAIN, input, NULL);
local_irq_restore(flags);
if (!hv_result_success(status)) {
hv_status_err(status, "HVCALL_CREATE_DEVICE_DOMAIN failed\n");
ida_free(&hv_iommu_device->domain_ids,
hv_domain->device_domain.domain_id.id);
}
return hv_result_to_errno(status);
}
static void hv_delete_device_domain(struct hv_iommu_domain *hv_domain)
{
u64 status;
unsigned long flags;
struct hv_input_delete_device_domain *input;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->device_domain = hv_domain->device_domain;
status = hv_do_hypercall(HVCALL_DELETE_DEVICE_DOMAIN, input, NULL);
local_irq_restore(flags);
if (!hv_result_success(status))
hv_status_err(status, "HVCALL_DELETE_DEVICE_DOMAIN failed\n");
ida_free(&hv_domain->hv_iommu->domain_ids,
hv_domain->device_domain.domain_id.id);
}
static int
hv_configure_device_domain(struct hv_iommu_domain *hv_domain,
const struct hv_device_domain_settings *settings)
{
u64 status;
unsigned long flags;
struct hv_input_configure_device_domain *input;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->device_domain = hv_domain->device_domain;
input->settings = *settings;
status = hv_do_hypercall(HVCALL_CONFIGURE_DEVICE_DOMAIN, input, NULL);
local_irq_restore(flags);
if (!hv_result_success(status))
hv_status_err(status,
"HVCALL_CONFIGURE_DEVICE_DOMAIN failed\n");
return hv_result_to_errno(status);
}
static int
hv_create_configure_device_domain(struct hv_iommu_domain *hv_domain,
const struct hv_device_domain_settings *settings)
{
int ret;
ret = hv_create_device_domain(hv_domain);
if (ret)
return ret;
ret = hv_configure_device_domain(hv_domain, settings);
if (ret)
hv_delete_device_domain(hv_domain);
return ret;
}
static bool hv_iommu_capable(struct device *dev, enum iommu_cap cap)
{
return cap == IOMMU_CAP_CACHE_COHERENCY;
}
static void hv_flush_device_domain(struct hv_iommu_domain *hv_domain)
{
u64 status;
unsigned long flags;
struct hv_input_flush_device_domain *input;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->device_domain = hv_domain->device_domain;
status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN, input, NULL);
local_irq_restore(flags);
WARN_ONCE(!hv_result_success(status),
"HVCALL_FLUSH_DEVICE_DOMAIN failed: %#llx (%s)\n",
status, hv_result_to_string(status));
}
static int hv_iommu_attach_dev(struct iommu_domain *domain, struct device *dev,
struct iommu_domain *old)
{
u64 status;
u32 prefix;
unsigned long flags;
struct pci_dev *pdev;
struct hv_input_attach_device_domain *input;
struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain);
int pci_domain;
int ret;
pdev = to_pci_dev(dev);
dev_dbg(dev, "attaching to domain %d\n",
hv_domain->device_domain.domain_id.id);
pci_domain = pci_domain_nr(pdev->bus);
ret = hv_pci_lookup_dev_id(pci_domain, &prefix);
if (ret) {
dev_err(dev,
"no logical device ID registered for PCI domain %04x\n",
pci_domain);
return ret;
}
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->device_domain = hv_domain->device_domain;
input->device_id.as_uint64 = (u64)prefix | PCI_FUNC(pdev->devfn);
status = hv_do_hypercall(HVCALL_ATTACH_DEVICE_DOMAIN, input, NULL);
local_irq_restore(flags);
if (!hv_result_success(status) &&
hv_result(status) != HV_STATUS_DEVICE_ALREADY_IN_DOMAIN) {
hv_status_err(status, "HVCALL_ATTACH_DEVICE_DOMAIN failed\n");
return hv_result_to_errno(status);
}
if (domain != &hv_blocking_domain.domain &&
!pdev->ats_enabled &&
hv_iommu_ats_supported(hv_domain->hv_iommu->cap) &&
pci_ats_supported(pdev))
pci_enable_ats(pdev, PAGE_SHIFT);
return 0;
}
static int hv_iommu_blocking_attach_dev(struct iommu_domain *domain,
struct device *dev,
struct iommu_domain *old)
{
struct pci_dev *pdev = to_pci_dev(dev);
int ret;
if (pdev->ats_enabled)
pci_disable_ats(pdev);
ret = hv_iommu_attach_dev(domain, dev, old);
/*
* Attaching to the blocking domain only asks the hypervisor to
* disable translation and IOPF for the device, so it cannot fail
* unless there is a driver or hypervisor bug. Return the hypercall
* status rather than 0 so that a failure on the DMA ownership claim
* path (VFIO/iommufd) fails the claim instead of leaving the device
* unblocked. WARN since such a failure indicates a bug.
*/
WARN_ON(ret);
return ret;
}
static int
hv_iommu_get_dev_property(struct device *dev, u32 code,
struct hv_output_get_logical_device_property *prop)
{
u64 status;
u32 prefix;
unsigned long flags;
int ret;
struct pci_dev *pdev = to_pci_dev(dev);
struct hv_input_get_logical_device_property *input;
struct hv_output_get_logical_device_property *output;
ret = hv_pci_lookup_dev_id(pci_domain_nr(pdev->bus), &prefix);
if (ret)
return ret;
local_irq_save(flags);
/*
* The combined input and output fit in the per-CPU input page,
* so no separate output page is needed.
*/
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
output = (struct hv_output_get_logical_device_property *)(input + 1);
memset(input, 0, sizeof(*input));
input->partition_id = HV_PARTITION_ID_SELF;
input->logical_device_id = (u64)prefix | PCI_FUNC(pdev->devfn);
input->code = code;
status = hv_do_hypercall(HVCALL_GET_LOGICAL_DEVICE_PROPERTY,
input, output);
*prop = *output;
local_irq_restore(flags);
if (!hv_result_success(status))
hv_status_err(status,
"HVCALL_GET_LOGICAL_DEVICE_PROPERTY failed\n");
return hv_result_to_errno(status);
}
static struct iommu_device *hv_iommu_probe_device(struct device *dev)
{
struct hv_output_get_logical_device_property property = {0};
int ret;
if (!dev_is_pci(dev))
return ERR_PTR(-ENODEV);
ret = hv_iommu_get_dev_property(dev, HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU,
&property);
if (ret || !(property.device_iommu & HV_DEVICE_IOMMU_ENABLED))
return ERR_PTR(-ENODEV);
return &hv_iommu_device->iommu;
}
static struct iommu_group *hv_iommu_device_group(struct device *dev)
{
if (!dev_is_pci(dev))
return ERR_PTR(-ENODEV);
return pci_device_group(dev);
}
static int __init hv_initialize_static_domains(void)
{
/*
* Clearing translation_enabled bypasses stage-1 translation, so DMA
* addresses are used directly as GPAs. Hyper-V requires paging and
* blocked domains to keep translation_enabled set.
*/
const struct hv_device_domain_settings identity_settings = {
.flags.translation_enabled = 0,
};
const struct hv_device_domain_settings blocked_settings = {
.flags = {
.translation_enabled = 1,
.blocked = 1,
},
};
int ret;
/* Default stage-1 identity domain */
ret = hv_create_configure_device_domain(&hv_identity_domain,
&identity_settings);
if (ret)
return ret;
/* Default stage-1 blocked domain */
ret = hv_create_configure_device_domain(&hv_blocking_domain,
&blocked_settings);
if (ret)
goto delete_identity_domain;
return 0;
delete_identity_domain:
hv_delete_device_domain(&hv_identity_domain);
return ret;
}
static void hv_iommu_get_resv_regions(struct device *dev,
struct list_head *head)
{
iommu_add_resv_x86_msi_region(head);
}
static void hv_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
hv_flush_device_domain(to_hv_iommu_domain(domain));
}
/*
* Calculate the minimal power-of-two aligned range that covers [start, end]
* (end is inclusive). Returns a single (page_number, page_mask_shift)
* descriptor that may over-flush when the range is not naturally aligned.
*/
static void
hv_iommu_calc_flush_range(unsigned long start, unsigned long end,
union hv_iommu_flush_va *va)
{
unsigned long aligned_start;
unsigned int sz_lg2;
sz_lg2 = max(fls_long(start ^ end), HV_HYP_PAGE_SHIFT);
/*
* A valid IOVA range shall not span bit 63. Use the maximum mask
* so the host can safely perform a full flush.
*/
if (WARN_ON_ONCE(sz_lg2 == BITS_PER_LONG))
aligned_start = 0;
else
aligned_start =
start & GENMASK(BITS_PER_LONG - 1, sz_lg2);
va->page_number = aligned_start >> HV_HYP_PAGE_SHIFT;
va->page_mask_shift = sz_lg2 - HV_HYP_PAGE_SHIFT;
}
static void hv_flush_device_domain_list(struct hv_iommu_domain *hv_domain,
struct iommu_iotlb_gather *iotlb_gather)
{
u64 status;
unsigned long flags;
struct hv_input_flush_device_domain_list *input;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
/* Clear the fixed header and the single range entry. */
memset(input, 0, struct_size(input, iova_list, 1));
input->device_domain = hv_domain->device_domain;
input->flags |= HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT;
hv_iommu_calc_flush_range(iotlb_gather->start, iotlb_gather->end,
&input->iova_list[0]);
status = hv_do_rep_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN_LIST,
1, 0, input, NULL);
if (WARN_ONCE(!hv_result_success(status),
"HVCALL_FLUSH_DEVICE_DOMAIN_LIST failed: %#llx (%s)\n",
status, hv_result_to_string(status))) {
/* Page-selective flush failed, fall back to full flush. */
struct hv_input_flush_device_domain *flush_all = (void *)input;
memset(flush_all, 0, sizeof(*flush_all));
flush_all->device_domain = hv_domain->device_domain;
status = hv_do_hypercall(HVCALL_FLUSH_DEVICE_DOMAIN,
flush_all, NULL);
WARN(!hv_result_success(status),
"HVCALL_FLUSH_DEVICE_DOMAIN fallback also failed: %#llx (%s)\n",
status, hv_result_to_string(status));
}
local_irq_restore(flags);
}
static void hv_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *iotlb_gather)
{
hv_flush_device_domain_list(to_hv_iommu_domain(domain), iotlb_gather);
/*
* The hypercall also invalidates non-leaf page-walk caches, so
* detached page-table pages can now be freed.
*/
iommu_put_pages_list(&iotlb_gather->freelist);
}
static void hv_iommu_paging_domain_free(struct iommu_domain *domain)
{
struct hv_iommu_domain *hv_domain = to_hv_iommu_domain(domain);
/* Free all remaining mappings */
pt_iommu_deinit(&hv_domain->pt_iommu);
hv_delete_device_domain(hv_domain);
kfree(hv_domain);
}
static const struct iommu_domain_ops hv_iommu_identity_domain_ops = {
.attach_dev = hv_iommu_attach_dev,
};
static const struct iommu_domain_ops hv_iommu_blocking_domain_ops = {
.attach_dev = hv_iommu_blocking_attach_dev,
};
static const struct iommu_domain_ops hv_iommu_paging_domain_ops = {
.attach_dev = hv_iommu_attach_dev,
IOMMU_PT_DOMAIN_OPS(x86_64),
.flush_iotlb_all = hv_iommu_flush_iotlb_all,
.iotlb_sync = hv_iommu_iotlb_sync,
.free = hv_iommu_paging_domain_free,
};
static struct iommu_domain *hv_iommu_domain_alloc_paging(struct device *dev)
{
int ret;
struct hv_iommu_domain *hv_domain;
struct pt_iommu_x86_64_hw_info pt_info;
struct pt_iommu_x86_64_cfg cfg = {};
struct hv_device_domain_settings settings = {
.flags = {
.translation_enabled = 1,
},
};
hv_domain = kzalloc_obj(*hv_domain, GFP_KERNEL);
if (!hv_domain)
return ERR_PTR(-ENOMEM);
hv_domain->pt_iommu.nid = dev_to_node(dev);
cfg.common.hw_max_vasz_lg2 = hv_iommu_device->max_iova_width;
cfg.common.hw_max_oasz_lg2 = 52;
cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE);
/*
* Hyper-V S1 domains use a 4-level root for IOVA widths up to
* 48 bits. A 5-level root is used only for wider apertures when
* HV_IOMMU_CAP_S1_5LVL is advertised.
*/
cfg.top_level = (hv_iommu_device->max_iova_width > 48) ? 4 : 3;
ret = pt_iommu_x86_64_init(&hv_domain->pt_iommu_x86_64, &cfg, GFP_KERNEL);
if (ret)
goto err_free;
/* Constrain to page sizes the hypervisor supports */
hv_domain->domain.pgsize_bitmap &= hv_iommu_device->pgsize_bitmap;
hv_domain->domain.ops = &hv_iommu_paging_domain_ops;
pt_iommu_x86_64_hw_info(&hv_domain->pt_iommu_x86_64, &pt_info);
settings.page_table_root = pt_info.gcr3_pt;
settings.flags.first_stage_paging_mode = pt_info.levels == 5;
ret = hv_create_configure_device_domain(hv_domain, &settings);
if (ret)
goto err_pt_deinit;
return &hv_domain->domain;
err_pt_deinit:
pt_iommu_deinit(&hv_domain->pt_iommu);
err_free:
kfree(hv_domain);
return ERR_PTR(ret);
}
static struct iommu_ops hv_iommu_ops = {
.capable = hv_iommu_capable,
.domain_alloc_paging = hv_iommu_domain_alloc_paging,
.probe_device = hv_iommu_probe_device,
.device_group = hv_iommu_device_group,
.get_resv_regions = hv_iommu_get_resv_regions,
.owner = THIS_MODULE,
.identity_domain = &hv_identity_domain.domain,
.blocked_domain = &hv_blocking_domain.domain,
.release_domain = &hv_blocking_domain.domain,
};
static u64 hv_iommu_detect(struct hv_output_get_iommu_capabilities *cap)
{
u64 status;
unsigned long flags;
struct hv_input_get_iommu_capabilities *input;
struct hv_output_get_iommu_capabilities *output;
local_irq_save(flags);
/*
* The combined input and output fit in the per-CPU input page,
* so no separate output page is needed.
*/
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
output = (struct hv_output_get_iommu_capabilities *)(input + 1);
memset(input, 0, sizeof(*input));
input->partition_id = HV_PARTITION_ID_SELF;
status = hv_do_hypercall(HVCALL_GET_IOMMU_CAPABILITIES, input, output);
*cap = *output;
local_irq_restore(flags);
return status;
}
static void __init
hv_init_iommu_device(struct hv_iommu_dev *hv_iommu,
struct hv_output_get_iommu_capabilities *hv_iommu_cap)
{
ida_init(&hv_iommu->domain_ids);
hv_iommu->cap = hv_iommu_cap->iommu_cap;
hv_iommu->max_iova_width = hv_iommu_cap->max_iova_width;
if (!hv_iommu_5lvl_supported(hv_iommu->cap) &&
hv_iommu->max_iova_width > 48) {
pr_info("5-level paging not supported, limiting iova width to 48.\n");
hv_iommu->max_iova_width = 48;
}
hv_iommu->first_domain = HV_DEVICE_DOMAIN_ID_DEFAULT + 1;
hv_iommu->last_domain = HV_DEVICE_DOMAIN_ID_NULL - 1;
hv_iommu->pgsize_bitmap = hv_iommu_cap->pgsize_bitmap;
hv_iommu_device = hv_iommu;
}
int __init hv_iommu_init(void)
{
u64 status;
int ret = 0;
struct hv_iommu_dev *hv_iommu = NULL;
struct hv_output_get_iommu_capabilities hv_iommu_cap = {0};
if (no_iommu || iommu_detected)
return -ENODEV;
if (!hv_is_hyperv_initialized())
return -ENODEV;
status = hv_iommu_detect(&hv_iommu_cap);
if (!hv_result_success(status)) {
if (hv_result(status) == HV_STATUS_INVALID_HYPERCALL_CODE)
return -ENODEV;
hv_status_err(status, "HVCALL_GET_IOMMU_CAPABILITIES failed\n");
return hv_result_to_errno(status);
}
if (!hv_iommu_present(hv_iommu_cap.iommu_cap))
return -ENODEV;
if (!hv_iommu_s1_domain_supported(hv_iommu_cap.iommu_cap)) {
pr_err("stage-1 translation not supported: cap=%#llx\n",
hv_iommu_cap.iommu_cap);
return -ENODEV;
}
/*
* Require the base page size. The domain page-size bitmap is later
* restricted to the sizes supported by both iommupt and Hyper-V.
*/
if (!(hv_iommu_cap.pgsize_bitmap & PAGE_SIZE)) {
pr_err("unsupported page sizes: pgsize_bitmap=0x%llx\n",
hv_iommu_cap.pgsize_bitmap);
return -ENODEV;
}
iommu_detected = 1;
pci_request_acs();
hv_iommu = kzalloc_obj(*hv_iommu, GFP_KERNEL);
if (!hv_iommu)
return -ENOMEM;
hv_init_iommu_device(hv_iommu, &hv_iommu_cap);
ret = hv_initialize_static_domains();
if (ret)
goto err_free;
ret = iommu_device_sysfs_add(&hv_iommu->iommu, NULL, NULL, "%s", "hv-iommu");
if (ret) {
pr_err("iommu_device_sysfs_add failed: %d\n", ret);
goto err_delete_static_domains;
}
ret = iommu_device_register(&hv_iommu->iommu, &hv_iommu_ops, NULL);
if (ret) {
pr_err("iommu_device_register failed: %d\n", ret);
goto err_sysfs_remove;
}
pr_info("initialized: %u-bit IOVA aperture, page-size bitmap %#llx\n",
hv_iommu->max_iova_width, hv_iommu->pgsize_bitmap);
return 0;
err_sysfs_remove:
iommu_device_sysfs_remove(&hv_iommu->iommu);
err_delete_static_domains:
hv_delete_device_domain(&hv_blocking_domain);
hv_delete_device_domain(&hv_identity_domain);
err_free:
kfree(hv_iommu);
return ret;
}
drivers/iommu/hyperv/iommu.h
+44
View File
@@ -0,0 +1,44 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Hyper-V IOMMU driver.
*
* Copyright (C) 2024-2026, Microsoft, Inc.
*
*/
#ifndef _HYPERV_IOMMU_H
#define _HYPERV_IOMMU_H
struct hv_iommu_dev {
struct iommu_device iommu;
struct ida domain_ids;
/* Device configuration */
u8 max_iova_width;
u8 max_pasid_width;
u64 cap;
u64 pgsize_bitmap;
u64 first_domain;
u64 last_domain;
};
struct hv_iommu_domain {
union {
struct iommu_domain domain;
struct pt_iommu pt_iommu;
struct pt_iommu_x86_64 pt_iommu_x86_64;
};
struct hv_iommu_dev *hv_iommu;
struct hv_input_device_domain device_domain;
u64 pgsize_bitmap;
};
PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu, domain);
PT_IOMMU_CHECK_DOMAIN(struct hv_iommu_domain, pt_iommu_x86_64.iommu, domain);
#define to_hv_iommu_domain(d) \
container_of(d, struct hv_iommu_domain, domain)
#endif /* _HYPERV_IOMMU_H */
drivers/iommu/intel/dmar.c
+60 -10
View File
@@ -1344,6 +1344,49 @@ static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
(unsigned long long)desc->qw1);
}
static int qi_drain_remaining_descs(struct intel_iommu *iommu, int head,
int wait_index)
{
struct q_inval *qi = iommu->qi;
int shift = qi_shift(iommu);
struct qi_desc wait_desc = {};
struct qi_desc nop_desc = {};
cycles_t start;
nop_desc.qw0 = QI_IWD_FENCE | QI_IWD_TYPE;
wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) | QI_IWD_PRQ_DRAIN |
QI_IWD_STATUS_WRITE | QI_IWD_FENCE | QI_IWD_TYPE;
wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
while (head != wait_index) {
memcpy(qi->desc + (head << shift), &nop_desc, 1 << shift);
head = (head + 1) % QI_LENGTH;
}
WRITE_ONCE(qi->desc_status[wait_index], QI_IN_USE);
memcpy(qi->desc + (wait_index << shift), &wait_desc, 1 << shift);
/*
* Order the descriptor rewrites before the writel() that restarts
* the fetch engine.
*/
wmb();
writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
start = get_cycles();
while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) {
if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start)) {
pr_err("Timeout draining invalidation queue after IQE\n");
return -ETIMEDOUT;
}
cpu_relax();
}
return 0;
}
static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
{
u32 fault;
@@ -1366,18 +1409,25 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
* is cleared.
*/
if (fault & DMA_FSTS_IQE) {
int head_idx, ret;
head = readl(iommu->reg + DMAR_IQH_REG);
if ((head >> shift) == index) {
struct qi_desc *desc = qi->desc + head;
head_idx = (head >> shift) % QI_LENGTH;
/*
* The faulting descriptor can be anywhere within the current
* submission's range [index, wait_index]. Since the queue is
* circular, this submission may wrap around QI_LENGTH
* (index > wait_index in that case), so check both the
* non-wrapped and wrapped cases of the range.
*/
if (index <= wait_index ?
(head_idx >= index && head_idx <= wait_index) :
(head_idx >= index || head_idx <= wait_index)) {
ret = qi_drain_remaining_descs(iommu, head_idx, wait_index);
if (ret)
return ret;
/*
* desc->qw2 and desc->qw3 are either reserved or
* used by software as private data. We won't print
* out these two qw's for security consideration.
*/
memcpy(desc, qi->desc + (wait_index << shift),
1 << shift);
writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
pr_info("Invalidation Queue Error (IQE) cleared\n");
return -EINVAL;
}
drivers/iommu/intel/iommu.c
+138 -29
View File
@@ -39,8 +39,6 @@
#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
#define IOAPIC_RANGE_START (0xfee00000)
#define IOAPIC_RANGE_END (0xfeefffff)
#define IOVA_START_ADDR (0x1000)
#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
@@ -1452,12 +1450,103 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu)
}
}
/*
* Reserve a domain ID inherited from the previous kernel so that it is not
* handed out again while the copied translation structures are still live.
*
* Returns 0 when the ID is reserved, was already reserved, or cannot be
* re-assigned, and a negative errno for a genuine allocation failure.
*/
static int reserve_domain_id(struct intel_iommu *iommu, int did)
{
int ret;
if (did < 0 || did >= iommu->max_domain_id)
return 0;
ret = ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
/*
* Devices sharing a domain share its ID, so the same ID is seen in
* more than one context entry; -ENOSPC merely reports that it is
* already reserved. On success the allocated ID is returned, which
* is not an error either.
*/
if (ret == -ENOSPC || ret >= 0)
return 0;
return ret;
}
/*
* Reserve the domain IDs used by a scalable mode context entry copied from
* the previous kernel.
*/
static int copy_pasid_table_dids(struct intel_iommu *iommu, struct context_entry *ce)
{
struct pasid_dir_entry *dir;
unsigned long dir_size;
phys_addr_t dir_phys;
int ret = 0;
int i, j;
dir_phys = ce->lo & VTD_PAGE_MASK;
if (!dir_phys)
return 0;
dir_size = get_pasid_dir_size(ce);
dir = memremap(dir_phys, dir_size * sizeof(*dir), MEMREMAP_WB);
if (!dir)
return -ENOMEM;
for (i = 0; i < dir_size; i++) {
struct pasid_entry *table;
phys_addr_t table_phys;
if (!pasid_pde_is_present(&dir[i]))
continue;
/*
* Do not use get_pasid_table_from_pde(); that returns a
* phys_to_virt() pointer, which is not valid for memory
* owned by the previous kernel.
*/
table_phys = READ_ONCE(dir[i].val) & PDE_PFN_MASK;
if (!table_phys)
continue;
/* A PASID table is one page: PASID_TBL_ENTRIES * 64 bytes. */
table = memremap(table_phys, PAGE_SIZE, MEMREMAP_WB);
if (!table) {
ret = -ENOMEM;
goto out;
}
for (j = 0; j < PASID_TBL_ENTRIES; j++) {
if (!pasid_pte_is_present(&table[j]))
continue;
ret = reserve_domain_id(iommu, pasid_get_domain_id(&table[j]));
if (ret) {
memunmap(table);
goto out;
}
}
memunmap(table);
}
out:
memunmap(dir);
return ret;
}
static int copy_context_table(struct intel_iommu *iommu,
struct root_entry *old_re,
struct context_entry **tbl,
int bus, bool ext)
{
int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did;
int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0;
struct context_entry *new_ce = NULL, ce;
struct context_entry *old_ce = NULL;
struct root_entry re;
@@ -1519,10 +1608,20 @@ static int copy_context_table(struct intel_iommu *iommu,
if (!context_present(&ce))
continue;
did = context_domain_id(&ce);
if (did >= 0 && did < iommu->max_domain_id)
ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
/*
* The context entry only holds a domain ID in legacy mode.
* In scalable mode the IDs are in the PASID table entries.
*/
if (ext)
ret = copy_pasid_table_dids(iommu, &ce);
else
ret = reserve_domain_id(iommu, context_domain_id(&ce));
if (ret) {
/* Not yet published through @tbl, so free it here. */
iommu_free_pages(new_ce);
new_ce = NULL;
goto out_unmap;
}
set_context_copied(iommu, bus, devfn);
new_ce[idx] = ce;
@@ -1591,7 +1690,7 @@ static int copy_translation_tables(struct intel_iommu *iommu)
if (ret) {
pr_err("%s: Failed to copy context table for bus %d\n",
iommu->name, bus);
continue;
goto err_free_ctxt_tbls;
}
}
@@ -1623,11 +1722,27 @@ static int copy_translation_tables(struct intel_iommu *iommu)
memunmap(old_rt);
return 0;
err_free_ctxt_tbls:
/*
* None of these tables have been linked into iommu->root_entry yet,
* so they are unreachable and must be freed here.
*/
for (bus = 0; bus < ctxt_table_entries; bus++)
iommu_free_pages(ctxt_tbls[bus]);
kfree(ctxt_tbls);
out_unmap:
memunmap(old_rt);
err_free_bitmap:
bitmap_free(iommu->copied_tables);
iommu->copied_tables = NULL;
/*
* Only reservations taken from the old context entries can be in the
* ida at this point; no domain has been allocated on this IOMMU yet.
* ida_destroy() empties it and leaves it ready for reuse.
*/
ida_destroy(&iommu->domain_ida);
return ret;
}
@@ -2782,9 +2897,7 @@ static int blocking_domain_attach_dev(struct iommu_domain *domain,
struct device *dev,
struct iommu_domain *old)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev);
iopf_for_domain_remove(old, dev);
device_block_translation(dev);
return 0;
}
@@ -2911,10 +3024,10 @@ static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu,
*top_level = 4;
return min(57, mgaw);
} else if (mgaw > 39 && sagaw >= BIT(2)) {
*top_level = 3 + ffs(sagaw >> 3);
*top_level = 2 + ffs(sagaw >> 2);
return min(48, mgaw);
} else if (mgaw > 30 && sagaw >= BIT(1)) {
*top_level = 2 + ffs(sagaw >> 2);
*top_level = 1 + ffs(sagaw >> 1);
return min(39, mgaw);
}
return 0;
@@ -3412,7 +3525,6 @@ static void intel_iommu_get_resv_regions(struct device *device,
struct list_head *head)
{
int prot = DMA_PTE_READ | DMA_PTE_WRITE;
struct iommu_resv_region *reg;
struct dmar_rmrr_unit *rmrr;
struct device *i_dev;
int i;
@@ -3448,6 +3560,7 @@ static void intel_iommu_get_resv_regions(struct device *device,
#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
if (dev_is_pci(device)) {
struct pci_dev *pdev = to_pci_dev(device);
struct iommu_resv_region *reg;
if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
@@ -3459,12 +3572,7 @@ static void intel_iommu_get_resv_regions(struct device *device,
}
#endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
0, IOMMU_RESV_MSI, GFP_KERNEL);
if (!reg)
return;
list_add_tail(&reg->list, head);
iommu_add_resv_x86_msi_region(head);
}
static struct iommu_group *intel_iommu_device_group(struct device *dev)
@@ -3870,6 +3978,14 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
if (dev_is_real_dma_subdevice(dev))
return 0;
if (sm_supported(iommu))
ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
else
ret = device_setup_pass_through(dev);
if (ret)
return ret;
/*
* The identity domain has no iopf_handler, so no IOPF reference is
* taken for it. The reference held by the old domain must still be
@@ -3877,16 +3993,9 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
* not affect the IOPF reference count.
*/
iopf_for_domain_remove(old, dev);
info->domain_attached = true;
if (sm_supported(iommu))
ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
else
ret = device_setup_pass_through(dev);
if (!ret)
info->domain_attached = true;
return ret;
return 0;
}
static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
drivers/iommu/intel/iommu.h
+13 -4
View File
@@ -1105,12 +1105,20 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
unsigned int size_order,
struct qi_desc *desc)
{
unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
QI_DEV_IOTLB_PFSID(pfsid);
/*
* The widest range the descriptor can express is a full flush, encoded
* by making bit 63 the least significant zero bit of ADDR, that is
* @size_order == 64 - VTD_PAGE_SHIFT. Bound @size_order there so that
* a caller passing something larger cannot produce an out-of-range
* shift below.
*/
if (size_order > 64 - VTD_PAGE_SHIFT)
size_order = 64 - VTD_PAGE_SHIFT;
/*
* If S bit is 0, we only flush a single page. If S bit is set,
* The least significant zero bit indicates the invalidation address
@@ -1120,7 +1128,8 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
* Max Invs Pending (MIP) is set to 0 for now until we have DIT in
* ECAP.
*/
if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
if (size_order < 64 - VTD_PAGE_SHIFT &&
!IS_ALIGNED(addr, BIT_ULL(VTD_PAGE_SHIFT + size_order)))
pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
addr, size_order);
@@ -1136,7 +1145,7 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
VTD_PAGE_SHIFT);
/* Clear size_order bit to indicate size */
desc->qw1 &= ~mask;
desc->qw1 &= ~BIT_ULL(VTD_PAGE_SHIFT + size_order - 1);
/* Set the S bit to indicate flushing more than 1 page */
desc->qw1 |= QI_DEV_EIOTLB_SIZE;
}
drivers/iommu/intel/nested.c
+2
View File
@@ -59,6 +59,8 @@ static int intel_nested_attach_dev(struct iommu_domain *domain,
if (ret)
goto disable_iopf;
iopf_for_domain_remove(old, dev);
info->domain = dmar_domain;
info->domain_attached = true;
spin_lock_irqsave(&dmar_domain->lock, flags);
drivers/iommu/iommu-priv.h
+4
View File
@@ -30,6 +30,10 @@ static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwsp
void iommu_fwspec_free(struct device *dev);
int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain);
struct iommu_domain *iommu_get_dma_domain(struct device *dev);
int iommu_device_register_bus(struct iommu_device *iommu,
const struct iommu_ops *ops,
const struct bus_type *bus,
drivers/iommu/iommu-sva.c
+3 -11
View File
@@ -11,7 +11,6 @@
#include "iommu-priv.h"
static DEFINE_MUTEX(iommu_sva_lock);
static bool iommu_sva_present;
static LIST_HEAD(iommu_sva_mms);
static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
struct mm_struct *mm);
@@ -137,11 +136,9 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
goto out_free_domain;
domain->users = 1;
if (list_empty(&iommu_mm->sva_domains)) {
if (list_empty(&iommu_sva_mms))
iommu_sva_present = true;
if (list_empty(&iommu_mm->sva_domains))
list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms);
}
list_add(&domain->next, &iommu_mm->sva_domains);
out:
refcount_set(&handle->users, 1);
@@ -182,11 +179,8 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
if (--domain->users == 0) {
list_del(&domain->next);
if (list_empty(&iommu_mm->sva_domains)) {
if (list_empty(&iommu_mm->sva_domains))
list_del(&iommu_mm->mm_list_elm);
if (list_empty(&iommu_sva_mms))
iommu_sva_present = false;
}
iommu_domain_free(domain);
}
@@ -334,8 +328,6 @@ void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end)
struct iommu_mm_data *iommu_mm;
guard(mutex)(&iommu_sva_lock);
if (!iommu_sva_present)
return;
list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm)
mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end);
drivers/iommu/iommu.c
+4
View File
@@ -4243,6 +4243,10 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
if (!group)
return 0;
ret = iommu_deferred_attach(dev, iommu_get_domain_for_dev(dev));
if (ret)
return ret;
mutex_lock(&group->mutex);
/* An IDENTITY domain must pass through */
if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
drivers/iommu/riscv/iommu.c
+12
View File
@@ -16,6 +16,7 @@
#include <linux/acpi_rimt.h>
#include <linux/compiler.h>
#include <linux/crash_dump.h>
#include <linux/dma-map-ops.h>
#include <linux/init.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
@@ -1444,6 +1445,16 @@ static struct iommu_group *riscv_iommu_device_group(struct device *dev)
return generic_device_group(dev);
}
static bool riscv_iommu_capable(struct device *dev, enum iommu_cap cap)
{
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
return dev_is_dma_coherent(dev);
default:
return false;
}
}
static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
{
return iommu_fwspec_add_ids(dev, args->args, 1);
@@ -1505,6 +1516,7 @@ static void riscv_iommu_release_device(struct device *dev)
static const struct iommu_ops riscv_iommu_ops = {
.of_xlate = riscv_iommu_of_xlate,
.capable = riscv_iommu_capable,
.identity_domain = &riscv_iommu_identity_domain,
.blocked_domain = &riscv_iommu_blocking_domain,
.release_domain = &riscv_iommu_blocking_domain,
drivers/pci/controller/pci-hyperv.c
+16 -5
View File
@@ -641,10 +641,7 @@ static void hv_irq_retarget_interrupt(struct irq_data *data)
params->int_entry.source = HV_INTERRUPT_SOURCE_MSI;
params->int_entry.msi_entry.address.as_uint32 = int_desc->address & 0xffffffff;
params->int_entry.msi_entry.data.as_uint32 = int_desc->data;
params->device_id = (hbus->hdev->dev_instance.b[5] << 24) |
(hbus->hdev->dev_instance.b[4] << 16) |
(hbus->hdev->dev_instance.b[7] << 8) |
(hbus->hdev->dev_instance.b[6] & 0xf8) |
params->device_id = hv_build_logical_dev_id_prefix(hbus->hdev) |
PCI_FUNC(pdev->devfn);
params->int_target.vector = hv_msi_get_int_vector(data);
@@ -3742,6 +3739,7 @@ static int hv_pci_probe(struct hv_device *hdev,
struct hv_pcibus_device *hbus;
int ret, dom;
u16 dom_req;
u32 prefix;
char *name;
bridge = devm_pci_alloc_host_bridge(&hdev->device, 0);
@@ -3884,13 +3882,22 @@ static int hv_pci_probe(struct hv_device *hdev,
hbus->state = hv_pcibus_probed;
ret = create_root_hv_pci_bus(hbus);
/* Register the bus before scanning any devices on it. */
prefix = hv_build_logical_dev_id_prefix(hdev);
ret = hv_pci_register_dev_id(dom, prefix);
if (ret)
goto free_windows;
ret = create_root_hv_pci_bus(hbus);
if (ret)
goto unregister_pci_bus;
mutex_unlock(&hbus->state_lock);
return 0;
unregister_pci_bus:
hv_pci_unregister_dev_id(dom);
free_windows:
hv_pci_free_bridge_windows(hbus);
exit_d0:
@@ -4004,6 +4011,8 @@ static void hv_pci_remove(struct hv_device *hdev)
hbus = hv_get_drvdata(hdev);
if (hbus->state == hv_pcibus_installed) {
int dom = hbus->bridge->domain_nr;
tasklet_disable(&hdev->channel->callback_event);
hbus->state = hv_pcibus_removing;
tasklet_enable(&hdev->channel->callback_event);
@@ -4021,6 +4030,8 @@ static void hv_pci_remove(struct hv_device *hdev)
hv_pci_remove_slots(hbus);
pci_remove_root_bus(hbus->bridge->bus);
pci_unlock_rescan_remove();
hv_pci_unregister_dev_id(dom);
}
hv_pci_bus_exit(hdev, false);
include/asm-generic/mshyperv.h
+4
View File
@@ -73,6 +73,10 @@ extern enum hv_partition_type hv_curr_partition_type;
extern void * __percpu *hyperv_pcpu_input_arg;
extern void * __percpu *hyperv_pcpu_output_arg;
int hv_pci_register_dev_id(int pci_domain_nr, u32 logical_dev_id_prefix);
void hv_pci_unregister_dev_id(int pci_domain_nr);
int hv_pci_lookup_dev_id(int pci_domain_nr, u32 *prefix);
u64 hv_do_hypercall(u64 control, void *inputaddr, void *outputaddr);
u64 hv_do_fast_hypercall8(u16 control, u64 input8);
u64 hv_do_fast_hypercall16(u16 control, u64 input1, u64 input2);
include/hyperv/hvgdk_mini.h
+10
View File
@@ -37,6 +37,7 @@ struct hv_u128 {
#define HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED 0x20
#define HV_STATUS_INVALID_LP_INDEX 0x41
#define HV_STATUS_INVALID_REGISTER_VALUE 0x50
#define HV_STATUS_DEVICE_ALREADY_IN_DOMAIN 0x66
#define HV_STATUS_OPERATION_FAILED 0x71
#define HV_STATUS_INSUFFICIENT_ROOT_MEMORY 0x73
#define HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY 0x75
@@ -486,10 +487,17 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */
#define HVCALL_GET_VP_INDEX_FROM_APIC_ID 0x009a
#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0
#define HVCALL_CREATE_DEVICE_DOMAIN 0x00b1
#define HVCALL_ATTACH_DEVICE_DOMAIN 0x00b2
#define HVCALL_SIGNAL_EVENT_DIRECT 0x00c0
#define HVCALL_POST_MESSAGE_DIRECT 0x00c1
#define HVCALL_DISPATCH_VP 0x00c2
#define HVCALL_DETACH_DEVICE_DOMAIN 0x00c4
#define HVCALL_DELETE_DEVICE_DOMAIN 0x00c5
#define HVCALL_GET_GPA_PAGES_ACCESS_STATES 0x00c9
#define HVCALL_CONFIGURE_DEVICE_DOMAIN 0x00ce
#define HVCALL_FLUSH_DEVICE_DOMAIN 0x00d0
#define HVCALL_FLUSH_DEVICE_DOMAIN_LIST 0x00d1
#define HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d7
#define HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS 0x00d8
#define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db
@@ -502,6 +510,8 @@ union hv_vp_assist_msr_contents { /* HV_REGISTER_VP_ASSIST_PAGE */
#define HVCALL_MMIO_READ 0x0106
#define HVCALL_MMIO_WRITE 0x0107
#define HVCALL_DISABLE_HYP_EX 0x010f
#define HVCALL_GET_IOMMU_CAPABILITIES 0x0125
#define HVCALL_GET_LOGICAL_DEVICE_PROPERTY 0x0127
#define HVCALL_MAP_STATS_PAGE2 0x0131
/* HV_HYPERCALL_INPUT */
include/hyperv/hvhdk_mini.h
+155
View File
@@ -548,4 +548,159 @@ union hv_device_id { /* HV_DEVICE_ID */
} acpi;
} __packed;
/* Device domain types */
#define HV_DEVICE_DOMAIN_TYPE_S1 1 /* HV_DEVICE_DOMAIN_ID_TYPE_S1 */
/* ID for default domain and NULL domain */
#define HV_DEVICE_DOMAIN_ID_DEFAULT 0
#define HV_DEVICE_DOMAIN_ID_NULL 0xFFFFFFFFULL
union hv_device_domain_id { /* HV_DEVICE_DOMAIN_ID */
u64 as_uint64;
struct {
u32 type: 4;
u32 reserved: 28;
u32 id;
} __packed;
};
struct hv_input_device_domain { /* HV_INPUT_DEVICE_DOMAIN */
u64 partition_id;
union hv_input_vtl owner_vtl;
u8 padding[7];
union hv_device_domain_id domain_id;
} __packed;
/* HV_CREATE_DEVICE_DOMAIN_FLAGS */
union hv_create_device_domain_flags {
u32 as_uint32;
struct {
u32 forward_progress_required: 1;
u32 inherit_owning_vtl: 1;
u32 reserved: 30;
} __packed;
};
/* HV_INPUT_CREATE_DEVICE_DOMAIN */
struct hv_input_create_device_domain {
struct hv_input_device_domain device_domain;
union hv_create_device_domain_flags create_device_domain_flags;
u32 padding;
} __packed;
/* HV_INPUT_DELETE_DEVICE_DOMAIN */
struct hv_input_delete_device_domain {
struct hv_input_device_domain device_domain;
} __packed;
/* HV_INPUT_ATTACH_DEVICE_DOMAIN */
struct hv_input_attach_device_domain {
struct hv_input_device_domain device_domain;
union hv_device_id device_id;
} __packed;
/* HV_INPUT_DETACH_DEVICE_DOMAIN */
struct hv_input_detach_device_domain {
u64 partition_id;
union hv_device_id device_id;
} __packed;
struct hv_device_domain_settings { /* HV_DEVICE_DOMAIN_SETTINGS */
struct {
/*
* Enable translations. If not enabled, all transaction bypass
* S1 translations.
*/
u64 translation_enabled: 1;
u64 blocked: 1;
/*
* First stage address translation paging mode:
* 0: 4-level paging (default)
* 1: 5-level paging
*/
u64 first_stage_paging_mode: 1;
u64 reserved: 61;
} flags;
/* Address of translation table */
u64 page_table_root;
} __packed;
/* HV_INPUT_CONFIGURE_DEVICE_DOMAIN */
struct hv_input_configure_device_domain {
struct hv_input_device_domain device_domain;
struct hv_device_domain_settings settings;
} __packed;
/* HV_INPUT_GET_IOMMU_CAPABILITIES */
struct hv_input_get_iommu_capabilities {
u64 partition_id;
u64 reserved;
} __packed;
/* HV_OUTPUT_GET_IOMMU_CAPABILITIES */
struct hv_output_get_iommu_capabilities {
u32 size;
u16 reserved;
u8 max_iova_width;
u8 max_pasid_width;
#define HV_IOMMU_CAP_PRESENT BIT_ULL(0)
#define HV_IOMMU_CAP_S2 BIT_ULL(1)
#define HV_IOMMU_CAP_S1 BIT_ULL(2)
#define HV_IOMMU_CAP_S1_5LVL BIT_ULL(3)
#define HV_IOMMU_CAP_PASID BIT_ULL(4)
#define HV_IOMMU_CAP_ATS BIT_ULL(5)
#define HV_IOMMU_CAP_PRI BIT_ULL(6)
u64 iommu_cap;
u64 pgsize_bitmap;
} __packed;
/* HV_LOGICAL_DEVICE_PROPERTY_CODE */
enum hv_logical_device_property_code {
HV_LOGICAL_DEVICE_PROPERTY_PVIOMMU = 10,
};
/* HV_INPUT_GET_LOGICAL_DEVICE_PROPERTY */
struct hv_input_get_logical_device_property {
u64 partition_id;
u64 logical_device_id;
/* Takes values from enum hv_logical_device_property_code. */
u32 code;
u32 reserved;
} __packed;
/* HV_OUTPUT_GET_LOGICAL_DEVICE_PROPERTY */
struct hv_output_get_logical_device_property {
#define HV_DEVICE_IOMMU_ENABLED BIT_ULL(0)
u64 device_iommu;
u64 reserved;
} __packed;
/* HV_INPUT_FLUSH_DEVICE_DOMAIN */
struct hv_input_flush_device_domain {
struct hv_input_device_domain device_domain;
u32 flags;
u32 reserved;
} __packed;
union hv_iommu_flush_va { /* HV_IOMMU_GVA_RANGE */
u64 as_uint64;
struct {
u64 page_mask_shift : 6;
u64 reserved : 6;
u64 page_number : 52;
};
} __packed;
/* HV_INPUT_FLUSH_DEVICE_DOMAIN_LIST */
struct hv_input_flush_device_domain_list {
struct hv_input_device_domain device_domain;
#define HV_FLUSH_DEVICE_DOMAIN_LIST_IOMMU_FORMAT BIT(0)
u32 flags;
u32 reserved;
union hv_iommu_flush_va iova_list[];
} __packed;
#endif /* _HV_HVHDK_MINI_H */
include/linux/amd-iommu.h
+12 -1
View File
@@ -11,11 +11,11 @@
#include <linux/types.h>
struct amd_iommu;
struct pci_dev;
#ifdef CONFIG_AMD_IOMMU
struct task_struct;
struct pci_dev;
extern void amd_iommu_detect(void);
@@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; }
static inline bool amd_iommu_sev_tio_supported(void) { return false; }
#endif
#ifdef CONFIG_AMD_IOMMU
int amd_iommu_enable_perfopt(struct pci_dev *pdev);
void amd_iommu_disable_perfopt(struct pci_dev *pdev);
#else
static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev)
{
return 0;
}
static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { }
#endif
#endif /* _ASM_X86_AMD_IOMMU_H */
include/linux/generic_pt/common.h
+6
View File
@@ -213,4 +213,10 @@ enum {
PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START,
};
struct pt_bcm2712 {
struct pt_common common;
u8 bigpage_lg2;
u8 superpage_lg2;
};
#endif
include/linux/generic_pt/iommu.h
+12
View File
@@ -346,6 +346,18 @@ struct pt_iommu_x86_64_hw_info {
IOMMU_FORMAT(x86_64, x86_64_pt);
struct pt_iommu_bcm2712_cfg {
struct pt_iommu_cfg common;
u8 bigpage_lg2;
u8 superpage_lg2;
};
struct pt_iommu_bcm2712_hw_info {
phys_addr_t pt_base;
};
IOMMU_FORMAT(bcm2712, bcm2712pt);
#undef IOMMU_PROTOTYPES
#undef IOMMU_FORMAT
#endif
include/linux/hyperv.h
+8
View File
@@ -1299,6 +1299,14 @@ struct hv_device {
#define device_to_hv_device(d) container_of_const(d, struct hv_device, device)
#define drv_to_hv_drv(d) container_of_const(d, struct hv_driver, driver)
static inline u32 hv_build_logical_dev_id_prefix(struct hv_device *hdev)
{
return ((u32)hdev->dev_instance.b[5] << 24) |
((u32)hdev->dev_instance.b[4] << 16) |
((u32)hdev->dev_instance.b[7] << 8) |
(hdev->dev_instance.b[6] & 0xf8u);
}
static inline void hv_set_drvdata(struct hv_device *dev, void *data)
{
dev_set_drvdata(&dev->device, data);
include/linux/iommu.h
-2
View File
@@ -900,7 +900,6 @@ int iommu_device_sysfs_add(struct iommu_device *iommu,
void iommu_device_sysfs_remove(struct iommu_device *iommu);
int iommu_device_link(struct iommu_device *iommu, struct device *link);
void iommu_device_unlink(struct iommu_device *iommu, struct device *link);
int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain);
static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
{
@@ -945,7 +944,6 @@ extern void iommu_detach_device(struct iommu_domain *domain,
struct device *dev);
extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev);
extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,