linux/drivers/resctrl/mpam_devices.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2025 Arm Ltd.

#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__

#include <linux/acpi.h>
#include <linux/atomic.h>
#include <linux/arm_mpam.h>
#include <linux/bitfield.h>
#include <linux/bitmap.h>
#include <linux/cacheinfo.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/errname.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/mailbox_client.h>
#include <linux/mutex.h>
#include <linux/platform_device.h>
#include <linux/printk.h>
#include <linux/property.h>
#include <linux/srcu.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/workqueue.h>

#include <acpi/pcc.h>
#include <acpi/acpi_io.h>

#include "mpam_internal.h"

/* Values for the T241 errata workaround */
#define T241_CHIPS_MAX			4
#define T241_CHIP_NSLICES		12
#define T241_SPARE_REG0_OFF		0x1b0000
#define T241_SPARE_REG1_OFF		0x1c0000
#define T241_CHIP_ID(phys)		FIELD_GET(GENMASK_ULL(44, 43), phys)
#define T241_SHADOW_REG_OFF(sidx, pid)	(0x360048 + (sidx) * 0x10000 + (pid) * 8)
#define SMCCC_SOC_ID_T241		0x036b0241
static void __iomem *t241_scratch_regs[T241_CHIPS_MAX];

/*
 * mpam_list_lock protects the SRCU lists when writing. Once the
 * mpam_enabled key is enabled these lists are read-only,
 * unless the error interrupt disables the driver.
 */
static DEFINE_MUTEX(mpam_list_lock);
static LIST_HEAD(mpam_all_msc);

struct srcu_struct mpam_srcu;

/* PCC channels might be serving multiple MSCs, so keep a refcounted list. */
static DEFINE_MUTEX(pcc_chan_list_lock);
static LIST_HEAD(pcc_chan_list);

static void mpam_pcc_chan_release(struct kref *ref)
{
	struct mpam_pcc_chan *cur = container_of(ref, struct mpam_pcc_chan,
						 refcount);

	pcc_mbox_free_channel(cur->pcc_chan);
	list_del(&cur->pcc_chans);
	mutex_destroy(&cur->pcc_chan_lock);
	kfree(cur);
}

static struct mpam_pcc_chan *mpam_pcc_chan_get(struct device *dev,
					       int subspace_id)
{
	struct mpam_pcc_chan *cur;

	guard(mutex)(&pcc_chan_list_lock);

	list_for_each_entry(cur, &pcc_chan_list, pcc_chans) {
		if (cur->subspace_id == subspace_id) {
			kref_get(&cur->refcount);

			return cur;
		}
	}

	cur = kzalloc_obj(*cur);
	if (!cur)
		return ERR_PTR(-ENOMEM);

	cur->pcc_cl.tx_block = true;

	cur->pcc_chan = pcc_mbox_request_channel(&cur->pcc_cl, subspace_id);
	if (IS_ERR(cur->pcc_chan)) {
		long err = PTR_ERR(cur->pcc_chan);

		kfree(cur);
		return ERR_PTR(err);
	}

	/*
	 * Timeout based on the "nominal latency" in us, from the
	 * PCC ACPI table. tx_tout is in ms.
	 * Add some margin here to be on the safe side.
	 */
	cur->pcc_cl.tx_tout = DIV_ROUND_UP(cur->pcc_chan->latency * 5, 1000);

	mutex_init(&cur->pcc_chan_lock);

	cur->subspace_id = subspace_id;
	kref_init(&cur->refcount);

	list_add_tail(&cur->pcc_chans, &pcc_chan_list);

	return cur;
}

static int mpam_pcc_chan_put(struct mpam_pcc_chan *pcc_chan)
{
	struct mpam_pcc_chan *cur, *tmp;

	if (!pcc_chan)
		return 0;

	guard(mutex)(&pcc_chan_list_lock);

	list_for_each_entry_safe(cur, tmp, &pcc_chan_list, pcc_chans) {
		if (cur == pcc_chan) {
			kref_put(&cur->refcount, mpam_pcc_chan_release);

			return 0;
		}
	}

	return -ENOENT;
}

/*
 * Number of MSCs that have been probed. Once all MSCs have been probed MPAM
 * can be enabled.
 */
static atomic_t mpam_num_msc;

static int mpam_cpuhp_state;
static DEFINE_MUTEX(mpam_cpuhp_state_lock);

/*
 * The smallest common values for any CPU or MSC in the system.
 * Generating traffic outside this range will result in screaming interrupts.
 */
u16 mpam_partid_max;
u8 mpam_pmg_max;
static bool partid_max_init, partid_max_published;
static DEFINE_SPINLOCK(partid_max_lock);

/*
 * mpam is enabled once all devices have been probed from CPU online callbacks,
 * scheduled via this work_struct. If access to an MSC depends on a CPU that
 * was not brought online at boot, this can happen surprisingly late.
 */
static DECLARE_WORK(mpam_enable_work, &mpam_enable);

/*
 * All mpam error interrupts indicate a software bug. On receipt, disable the
 * driver.
 */
static DECLARE_WORK(mpam_broken_work, &mpam_disable);

/* When mpam is disabled, the printed reason to aid debugging */
static char *mpam_disable_reason;
static int mpam_disable_errno;
static int mpam_disable_mpam_fb_err;

void mpam_fb_disable_mpam(int err, int mpam_fb_err)
{
	/* Prevent repeated calls when mpam_disable() does MSC accesses. */
	if (!mpam_is_enabled())
		return;

	mpam_disable_errno = err;
	mpam_disable_mpam_fb_err = mpam_fb_err;

	mpam_disable_reason = "MPAM-Fb error";
	schedule_work(&mpam_broken_work);
}

/*
 * Whether resctrl has been setup. Used by cpuhp in preference to
 * mpam_is_enabled(). The disable call after an error interrupt makes
 * mpam_is_enabled() false before the cpuhp callbacks are made.
 * Reads/writes should hold mpam_cpuhp_state_lock, (or be cpuhp callbacks).
 */
static bool mpam_resctrl_enabled;

/*
 * An MSC is a physical container for controls and monitors, each identified by
 * their RIS index. These share a base-address, interrupts and some MMIO
 * registers. A vMSC is a virtual container for RIS in an MSC that control or
 * monitor the same thing. Members of a vMSC are all RIS in the same MSC, but
 * not all RIS in an MSC share a vMSC.
 *
 * Components are a group of vMSC that control or monitor the same thing but
 * are from different MSC, so have different base-address, interrupts etc.
 * Classes are the set components of the same type.
 *
 * The features of a vMSC is the union of the RIS it contains.
 * The features of a Class and Component are the common subset of the vMSC
 * they contain.
 *
 * e.g. The system cache may have bandwidth controls on multiple interfaces,
 * for regulating traffic from devices independently of traffic from CPUs.
 * If these are two RIS in one MSC, they will be treated as controlling
 * different things, and will not share a vMSC/component/class.
 *
 * e.g. The L2 may have one MSC and two RIS, one for cache-controls another
 * for bandwidth. These two RIS are members of the same vMSC.
 *
 * e.g. The set of RIS that make up the L2 are grouped as a component. These
 * are sometimes termed slices. They should be configured the same, as if there
 * were only one.
 *
 * e.g. The SoC probably has more than one L2, each attached to a distinct set
 * of CPUs. All the L2 components are grouped as a class.
 *
 * When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list,
 * then linked via struct mpam_ris to a vmsc, component and class.
 * The same MSC may exist under different class->component->vmsc paths, but the
 * RIS index will be unique.
 */
LIST_HEAD(mpam_classes);

/* List of all objects that can be free()d after synchronise_srcu() */
static LLIST_HEAD(mpam_garbage);

static inline void init_garbage(struct mpam_garbage *garbage)
{
	init_llist_node(&garbage->llist);
}

#define add_to_garbage(x)				\
do {							\
	__typeof__(x) _x = (x);				\
	_x->garbage.to_free = _x;			\
	llist_add(&_x->garbage.llist, &mpam_garbage);	\
} while (0)

static void mpam_free_garbage(void)
{
	struct mpam_garbage *iter, *tmp;
	struct llist_node *to_free = llist_del_all(&mpam_garbage);

	if (!to_free)
		return;

	synchronize_srcu(&mpam_srcu);

	llist_for_each_entry_safe(iter, tmp, to_free, llist) {
		if (iter->pdev)
			devm_kfree(&iter->pdev->dev, iter->to_free);
		else
			kvfree(iter->to_free);
	}
}

/*
 * Once mpam is enabled, new requestors cannot further reduce the available
 * partid. Assert that the size is fixed, and new requestors will be turned
 * away. This is needed when walking over structures sized by PARTID.
 *
 * During mpam_disable() these structures are not fixed, but the MSC state
 * is still reset using whatever sizes have been discovered so far. As only
 * PARTID 0 will be used after mpam_disable(), any race would be benign.
 * Skip the check if a mpam_disable_reason has been set.
 */
static void mpam_assert_partid_sizes_fixed(void)
{
	if (!mpam_disable_reason)
		WARN_ON_ONCE(!partid_max_published);
}

static int __mpam_read_reg(struct mpam_msc *msc, u16 reg, u32 *res)
{

	if (msc->iface == MPAM_IFACE_PCC)
		return mpam_fb_send_read_request(msc, reg, res);

	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
	*res = readl_relaxed(msc->mapped_hwpage + reg);

	return 0;
}

static inline int _mpam_read_partsel_reg(struct mpam_msc *msc, u16 reg,
					 u32 *res)
{
	lockdep_assert_held_once(&msc->part_sel_lock);
	return __mpam_read_reg(msc, reg, res);
}

#define mpam_read_partsel_reg(msc, reg, res) _mpam_read_partsel_reg(msc, MPAMF_##reg, res)

static int __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val)
{

	if (msc->iface == MPAM_IFACE_PCC)
		return mpam_fb_send_write_request(msc, reg, val);

	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
	WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz);
	writel_relaxed(val, msc->mapped_hwpage + reg);

	return 0;
}

static inline int _mpam_write_partsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
{
	lockdep_assert_held_once(&msc->part_sel_lock);
	return __mpam_write_reg(msc, reg, val);
}

#define mpam_write_partsel_reg(msc, reg, val)  _mpam_write_partsel_reg(msc, MPAMCFG_##reg, val)

static inline int _mpam_read_monsel_reg(struct mpam_msc *msc, u16 reg,
					u32 *res)
{
	mpam_mon_sel_lock_held(msc);
	return __mpam_read_reg(msc, reg, res);
}

#define mpam_read_monsel_reg(msc, reg, res) _mpam_read_monsel_reg(msc, MSMON_##reg, res)

static inline int _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
{
	mpam_mon_sel_lock_held(msc);
	return __mpam_write_reg(msc, reg, val);
}

#define mpam_write_monsel_reg(msc, reg, val)   _mpam_write_monsel_reg(msc, MSMON_##reg, val)

static bool mpam_msc_check_aidr(struct mpam_msc *msc)
{
	u32 aidr, major, minor;

	if (__mpam_read_reg(msc, MPAMF_AIDR, &aidr))
		return false;

	major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr);
	minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr);
	/*
	 * v0.0 and >v2.x aren't supported, but anything else should be backward
	 * compatible to v0.1 or v1.0.
	 */
	if (!major && !minor)
		return false;
	if (major > 1)
		return false;

	return true;
}

static int mpam_msc_read_idr(struct mpam_msc *msc, u64 *res)
{
	u32 idr_high, idr_low;
	int ret;

	lockdep_assert_held(&msc->part_sel_lock);

	ret = mpam_read_partsel_reg(msc, IDR, &idr_low);
	if (ret)
		return ret;

	if (FIELD_GET(MPAMF_IDR_EXT, idr_low)) {
		ret = mpam_read_partsel_reg(msc, IDR + 4, &idr_high);
		if (ret)
			return ret;
	} else {
		idr_high = 0;
	}

	*res = ((u64)idr_high << 32) | idr_low;

	return 0;
}

static int mpam_msc_clear_esr(struct mpam_msc *msc)
{
	u32 esr_low;
	int ret;

	ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low);
	if (ret)
		return ret;

	if (!esr_low)
		return 0;

	/*
	 * Clearing the high/low bits of MPAMF_ESR can not be atomic.
	 * Clear the top half first, so that the pending error bits in the
	 * lower half prevent hardware from updating either half of the
	 * register.
	 */
	if (msc->has_extd_esr) {
		ret = __mpam_write_reg(msc, MPAMF_ESR + 4, 0);
		if (ret)
			return ret;
	}

	return __mpam_write_reg(msc, MPAMF_ESR, 0);
}

static int mpam_msc_read_esr(struct mpam_msc *msc, u64 *res)
{
	u32 esr_high, esr_low;
	int ret;

	ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low);
	if (ret)
		return ret;

	if (msc->has_extd_esr) {
		ret = __mpam_read_reg(msc, MPAMF_ESR + 4, &esr_high);
		if (ret)
			return ret;
	} else {
		esr_high = 0;
	}

	*res = ((u64)esr_high << 32) | esr_low;

	return 0;
}

static int __mpam_part_sel_raw(u32 partsel, struct mpam_msc *msc)
{
	lockdep_assert_held(&msc->part_sel_lock);

	return mpam_write_partsel_reg(msc, PART_SEL, partsel);
}

static int __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc)
{
	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) |
		      FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid);

	return __mpam_part_sel_raw(partsel, msc);
}

static int __mpam_intpart_sel(u8 ris_idx, u16 intpartid, struct mpam_msc *msc)
{
	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) |
		      FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, intpartid) |
		      MPAMCFG_PART_SEL_INTERNAL;

	return __mpam_part_sel_raw(partsel, msc);
}

int mpam_register_requestor(u16 partid_max, u8 pmg_max)
{
	guard(spinlock)(&partid_max_lock);
	if (!partid_max_init) {
		mpam_partid_max = partid_max;
		mpam_pmg_max = pmg_max;
		partid_max_init = true;
	} else if (!partid_max_published) {
		mpam_partid_max = min(mpam_partid_max, partid_max);
		mpam_pmg_max = min(mpam_pmg_max, pmg_max);
	} else {
		/* New requestors can't lower the values */
		if (partid_max < mpam_partid_max || pmg_max < mpam_pmg_max)
			return -EBUSY;
	}

	return 0;
}
EXPORT_SYMBOL(mpam_register_requestor);

static struct mpam_class *
mpam_class_alloc(u8 level_idx, enum mpam_class_types type)
{
	struct mpam_class *class;

	lockdep_assert_held(&mpam_list_lock);

	class = kzalloc_obj(*class);
	if (!class)
		return ERR_PTR(-ENOMEM);
	init_garbage(&class->garbage);

	INIT_LIST_HEAD_RCU(&class->components);
	/* Affinity is updated when ris are added */
	class->level = level_idx;
	class->type = type;
	INIT_LIST_HEAD_RCU(&class->classes_list);
	ida_init(&class->ida_csu_mon);
	ida_init(&class->ida_mbwu_mon);

	list_add_rcu(&class->classes_list, &mpam_classes);

	return class;
}

static void mpam_class_destroy(struct mpam_class *class)
{
	lockdep_assert_held(&mpam_list_lock);

	list_del_rcu(&class->classes_list);
	add_to_garbage(class);
}

static struct mpam_class *
mpam_class_find(u8 level_idx, enum mpam_class_types type)
{
	struct mpam_class *class;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry(class, &mpam_classes, classes_list) {
		if (class->type == type && class->level == level_idx)
			return class;
	}

	return mpam_class_alloc(level_idx, type);
}

static struct mpam_component *
mpam_component_alloc(struct mpam_class *class, int id)
{
	struct mpam_component *comp;

	lockdep_assert_held(&mpam_list_lock);

	comp = kzalloc_obj(*comp);
	if (!comp)
		return ERR_PTR(-ENOMEM);
	init_garbage(&comp->garbage);

	comp->comp_id = id;
	INIT_LIST_HEAD_RCU(&comp->vmsc);
	/* Affinity is updated when RIS are added */
	INIT_LIST_HEAD_RCU(&comp->class_list);
	comp->class = class;

	list_add_rcu(&comp->class_list, &class->components);

	return comp;
}

static void __destroy_component_cfg(struct mpam_component *comp);

static void mpam_component_destroy(struct mpam_component *comp)
{
	struct mpam_class *class = comp->class;

	lockdep_assert_held(&mpam_list_lock);

	__destroy_component_cfg(comp);

	list_del_rcu(&comp->class_list);
	add_to_garbage(comp);

	if (list_empty(&class->components))
		mpam_class_destroy(class);
}

static struct mpam_component *
mpam_component_find(struct mpam_class *class, int id)
{
	struct mpam_component *comp;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry(comp, &class->components, class_list) {
		if (comp->comp_id == id)
			return comp;
	}

	return mpam_component_alloc(class, id);
}

static struct mpam_vmsc *
mpam_vmsc_alloc(struct mpam_component *comp, struct mpam_msc *msc)
{
	struct mpam_vmsc *vmsc;

	lockdep_assert_held(&mpam_list_lock);

	vmsc = kzalloc_obj(*vmsc);
	if (!vmsc)
		return ERR_PTR(-ENOMEM);
	init_garbage(&vmsc->garbage);

	INIT_LIST_HEAD_RCU(&vmsc->ris);
	INIT_LIST_HEAD_RCU(&vmsc->comp_list);
	vmsc->comp = comp;
	vmsc->msc = msc;

	list_add_rcu(&vmsc->comp_list, &comp->vmsc);

	return vmsc;
}

static void mpam_vmsc_destroy(struct mpam_vmsc *vmsc)
{
	struct mpam_component *comp = vmsc->comp;

	lockdep_assert_held(&mpam_list_lock);

	list_del_rcu(&vmsc->comp_list);
	add_to_garbage(vmsc);

	if (list_empty(&comp->vmsc))
		mpam_component_destroy(comp);
}

static struct mpam_vmsc *
mpam_vmsc_find(struct mpam_component *comp, struct mpam_msc *msc)
{
	struct mpam_vmsc *vmsc;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
		if (vmsc->msc->id == msc->id)
			return vmsc;
	}

	return mpam_vmsc_alloc(comp, msc);
}

/*
 * The cacheinfo structures are only populated when CPUs are online.
 * This helper walks the acpi tables to include offline CPUs too.
 */
int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level,
				   cpumask_t *affinity)
{
	return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity);
}

/*
 * cpumask_of_node() only knows about online CPUs. This can't tell us whether
 * a class is represented on all possible CPUs.
 */
static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		if (node_id == cpu_to_node(cpu))
			cpumask_set_cpu(cpu, affinity);
	}
}

static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity,
				 enum mpam_class_types type,
				 struct mpam_class *class,
				 struct mpam_component *comp)
{
	int err;

	switch (type) {
	case MPAM_CLASS_CACHE:
		err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level,
						     affinity);
		if (err) {
			dev_warn_once(&msc->pdev->dev,
				      "Failed to determine CPU affinity\n");
			return err;
		}

		if (cpumask_empty(affinity))
			dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n");

		break;
	case MPAM_CLASS_MEMORY:
		get_cpumask_from_node_id(comp->comp_id, affinity);
		/* affinity may be empty for CPU-less memory nodes */
		break;
	case MPAM_CLASS_UNKNOWN:
		return 0;
	}

	cpumask_and(affinity, affinity, &msc->accessibility);

	return 0;
}

static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx,
				  enum mpam_class_types type, u8 class_id,
				  int component_id)
{
	int err;
	struct mpam_vmsc *vmsc;
	struct mpam_msc_ris *ris;
	struct mpam_class *class;
	struct mpam_component *comp;
	struct platform_device *pdev = msc->pdev;

	lockdep_assert_held(&mpam_list_lock);

	if (ris_idx > MPAM_MSC_MAX_NUM_RIS)
		return -EINVAL;

	if (test_and_set_bit(ris_idx, &msc->ris_idxs))
		return -EBUSY;

	ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL);
	if (!ris)
		return -ENOMEM;
	init_garbage(&ris->garbage);
	ris->garbage.pdev = pdev;

	class = mpam_class_find(class_id, type);
	if (IS_ERR(class))
		return PTR_ERR(class);

	comp = mpam_component_find(class, component_id);
	if (IS_ERR(comp)) {
		if (list_empty(&class->components))
			mpam_class_destroy(class);
		return PTR_ERR(comp);
	}

	vmsc = mpam_vmsc_find(comp, msc);
	if (IS_ERR(vmsc)) {
		if (list_empty(&comp->vmsc))
			mpam_component_destroy(comp);
		return PTR_ERR(vmsc);
	}

	err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp);
	if (err) {
		if (list_empty(&vmsc->ris))
			mpam_vmsc_destroy(vmsc);
		return err;
	}

	ris->ris_idx = ris_idx;
	INIT_LIST_HEAD_RCU(&ris->msc_list);
	INIT_LIST_HEAD_RCU(&ris->vmsc_list);
	ris->vmsc = vmsc;

	cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity);
	cpumask_or(&class->affinity, &class->affinity, &ris->affinity);
	list_add_rcu(&ris->vmsc_list, &vmsc->ris);
	list_add_rcu(&ris->msc_list, &msc->ris);

	return 0;
}

static void mpam_ris_destroy(struct mpam_msc_ris *ris)
{
	struct mpam_vmsc *vmsc = ris->vmsc;
	struct mpam_msc *msc = vmsc->msc;
	struct mpam_component *comp = vmsc->comp;
	struct mpam_class *class = comp->class;

	lockdep_assert_held(&mpam_list_lock);

	/*
	 * It is assumed affinities don't overlap. If they do the class becomes
	 * unusable immediately.
	 */
	cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity);
	cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity);
	clear_bit(ris->ris_idx, &msc->ris_idxs);
	list_del_rcu(&ris->msc_list);
	list_del_rcu(&ris->vmsc_list);
	add_to_garbage(ris);

	if (list_empty(&vmsc->ris))
		mpam_vmsc_destroy(vmsc);
}

int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
		    enum mpam_class_types type, u8 class_id, int component_id)
{
	int err;

	mutex_lock(&mpam_list_lock);
	err = mpam_ris_create_locked(msc, ris_idx, type, class_id,
				     component_id);
	mutex_unlock(&mpam_list_lock);
	if (err)
		mpam_free_garbage();

	return err;
}

static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc,
						   u8 ris_idx)
{
	int err;
	struct mpam_msc_ris *ris;

	lockdep_assert_held(&mpam_list_lock);

	if (!test_bit(ris_idx, &msc->ris_idxs)) {
		err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN,
					     0, 0);
		if (err)
			return ERR_PTR(err);
	}

	list_for_each_entry(ris, &msc->ris, msc_list) {
		if (ris->ris_idx == ris_idx)
			return ris;
	}

	return ERR_PTR(-ENOENT);
}

static int mpam_enable_quirk_nvidia_t241_1(struct mpam_msc *msc,
					   const struct mpam_quirk *quirk)
{
	s32 soc_id = arm_smccc_get_soc_id_version();
	struct resource *r;
	phys_addr_t phys;

	/*
	 * A mapping to a device other than the MSC is needed, check
	 * SOC_ID is  NVIDIA T241 chip (036b:0241)
	 */
	if (soc_id < 0 || soc_id != SMCCC_SOC_ID_T241)
		return -EINVAL;

	r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0);
	if (!r)
		return -EINVAL;

	/* Find the internal registers base addr from the CHIP ID */
	msc->t241_id = T241_CHIP_ID(r->start);
	phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) | 0x19000000ULL;

	t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M);
	if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id]))
		return -EINVAL;

	pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n");

	return 0;
}

static const struct mpam_quirk mpam_quirks[] = {
	{
		/* NVIDIA t241 erratum T241-MPAM-1 */
		.init       = mpam_enable_quirk_nvidia_t241_1,
		.iidr       = MPAM_IIDR_NVIDIA_T241,
		.iidr_mask  = MPAM_IIDR_MATCH_ONE,
		.workaround = T241_SCRUB_SHADOW_REGS,
	},
	{
		/* NVIDIA t241 erratum T241-MPAM-4 */
		.iidr       = MPAM_IIDR_NVIDIA_T241,
		.iidr_mask  = MPAM_IIDR_MATCH_ONE,
		.workaround = T241_FORCE_MBW_MIN_TO_ONE,
	},
	{
		/* NVIDIA t241 erratum T241-MPAM-6 */
		.iidr       = MPAM_IIDR_NVIDIA_T241,
		.iidr_mask  = MPAM_IIDR_MATCH_ONE,
		.workaround = T241_MBW_COUNTER_SCALE_64,
	},
	{
	/* ARM CMN-650 CSU erratum 3642720 */
	.iidr       = MPAM_IIDR_ARM_CMN_650,
	.iidr_mask  = MPAM_IIDR_MATCH_ONE,
	.workaround = IGNORE_CSU_NRDY,
	},
	{ NULL } /* Sentinel */
};

static void mpam_enable_quirks(struct mpam_msc *msc)
{
	const struct mpam_quirk *quirk;

	for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) {
		int err = 0;

		if (quirk->iidr != (msc->iidr & quirk->iidr_mask))
			continue;

		if (quirk->init)
			err = quirk->init(msc, quirk);

		if (err)
			continue;

		mpam_set_quirk(quirk->workaround, msc);
	}
}

/*
 * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour
 * of NRDY, software can use this bit for any purpose" - so hardware might not
 * implement this - but it isn't RES0.
 *
 * Try and see what values stick in this bit. If we can write either value,
 * its probably not implemented by hardware.
 */
static int mpam_ris_hw_probe_csu_nrdy(struct mpam_msc_ris *ris)
{
	u32 now, mon_sel, ctl_val;
	bool can_set, can_clear;
	struct mpam_msc *msc = ris->vmsc->msc;
	int ret;

	ACQUIRE(mon_sel_lock, guard)(msc);
	ret = ACQUIRE_ERR(mon_sel_lock, &guard);
	if (ret)
		return ret;

	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) |
		  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
	ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
	if (ret)
		return ret;

	/* Hardware might ignore nrdy if it's not enabled */
	ctl_val = MSMON_CFG_CSU_CTL_TYPE_CSU;
	ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID;
	ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG;
	ctl_val |= MSMON_CFG_x_CTL_EN;
	ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0);
	if (ret)
		return ret;
	ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
	if (ret)
		return ret;

	ret = _mpam_write_monsel_reg(msc, MSMON_CSU, MSMON___NRDY);
	if (ret)
		return ret;
	ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now);
	if (ret)
		return ret;
	can_set = now & MSMON___NRDY;

	ret = _mpam_write_monsel_reg(msc, MSMON_CSU, 0);
	if (ret)
		return ret;
	/* Configuration change to try and coax hardware into setting nrdy */
	ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0x1);
	if (ret)
		return ret;
	ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now);
	if (ret)
		return ret;
	can_clear = !(now & MSMON___NRDY);

	return (!can_set || !can_clear);
}

static int mpam_ris_hw_probe(struct mpam_msc_ris *ris)
{
	int fw_has_nrdy_err, err;
	struct mpam_msc *msc = ris->vmsc->msc;
	struct device *dev = &msc->pdev->dev;
	struct mpam_props *props = &ris->props;
	struct mpam_class *class = ris->vmsc->comp->class;

	lockdep_assert_held(&msc->probe_lock);
	lockdep_assert_held(&msc->part_sel_lock);

	/* Cache Capacity Partitioning */
	if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) {
		u32 ccap_features;

		err = mpam_read_partsel_reg(msc, CCAP_IDR, &ccap_features);
		if (err)
			return err;

		props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features);
		if (props->cmax_wd &&
		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM, ccap_features))
			mpam_set_feature(mpam_feat_cmax_softlim, props);

		if (props->cmax_wd &&
		    !FIELD_GET(MPAMF_CCAP_IDR_NO_CMAX, ccap_features))
			mpam_set_feature(mpam_feat_cmax_cmax, props);

		if (props->cmax_wd &&
		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CMIN, ccap_features))
			mpam_set_feature(mpam_feat_cmax_cmin, props);

		props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features);
		if (props->cassoc_wd &&
		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CASSOC, ccap_features))
			mpam_set_feature(mpam_feat_cmax_cassoc, props);
	}

	/* Cache Portion partitioning */
	if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) {
		u32 cpor_features;

		err = mpam_read_partsel_reg(msc, CPOR_IDR, &cpor_features);
		if (err)
			return err;

		props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features);
		if (props->cpbm_wd)
			mpam_set_feature(mpam_feat_cpor_part, props);
	}

	/* Memory bandwidth partitioning */
	if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) {
		u32 mbw_features;
		err = mpam_read_partsel_reg(msc, MBW_IDR, &mbw_features);
		if (err)
			return err;

		/* portion bitmap resolution */
		props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features);
		if (props->mbw_pbm_bits &&
		    FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features))
			mpam_set_feature(mpam_feat_mbw_part, props);

		props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features);

		/*
		 * The BWA_WD field can represent 0-63, but the control fields it
		 * describes have a maximum of 16 bits.
		 */
		props->bwa_wd = min(props->bwa_wd, 16);

		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features))
			mpam_set_feature(mpam_feat_mbw_max, props);

		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features))
			mpam_set_feature(mpam_feat_mbw_min, props);

		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features))
			mpam_set_feature(mpam_feat_mbw_prop, props);
	}

	/* Priority partitioning */
	if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) {
		u32 pri_features;
		err = mpam_read_partsel_reg(msc, PRI_IDR, &pri_features);
		if (err)
			return err;

		props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features);
		if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) {
			mpam_set_feature(mpam_feat_intpri_part, props);
			if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features))
				mpam_set_feature(mpam_feat_intpri_part_0_low, props);
		}

		props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features);
		if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) {
			mpam_set_feature(mpam_feat_dspri_part, props);
			if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features))
				mpam_set_feature(mpam_feat_dspri_part_0_low, props);
		}
	}

	/* Performance Monitoring */
	if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) {
		u32 msmon_features;

		err = mpam_read_partsel_reg(msc, MSMON_IDR, &msmon_features);
		if (err)
			return err;

		/*
		 * If the firmware max-nrdy-us property is missing, the
		 * CSU counters can't be used. Should we wait forever?
		 */
		fw_has_nrdy_err = device_property_read_u32(&msc->pdev->dev,
							   "arm,not-ready-us",
							   &msc->nrdy_usec);

		if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) {
			u32 csumonidr;

			err = mpam_read_partsel_reg(msc, CSUMON_IDR, &csumonidr);
			if (err)
				return err;

			props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr);
			if (props->num_csu_mon) {
				bool hw_managed;

				mpam_set_feature(mpam_feat_msmon_csu, props);

				if (FIELD_GET(MPAMF_CSUMON_IDR_HAS_XCL, csumonidr))
					mpam_set_feature(mpam_feat_msmon_csu_xcl, props);

				/* Is NRDY hardware managed? */
				err = mpam_ris_hw_probe_csu_nrdy(ris);
				if (err < 0)
					return err;
				hw_managed = err;

				/*
				 * Accept the missing firmware property if NRDY appears
				 * un-implemented.
				 */
				if (fw_has_nrdy_err && hw_managed)
					dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.");
			}
		}
		if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) {
			bool has_long;
			u32 mbwumon_idr;

			err = mpam_read_partsel_reg(msc, MBWUMON_IDR, &mbwumon_idr);
			if (err)
				return err;

			props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr);
			if (props->num_mbwu_mon) {
				mpam_set_feature(mpam_feat_msmon_mbwu, props);

				if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumon_idr))
					mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props);

				has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumon_idr);
				if (has_long) {
					if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumon_idr))
						mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props);
					else
						mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props);
				}

				mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props);
			}
		}
	}

	/*
	 * RIS with PARTID narrowing don't have enough storage for one
	 * configuration per PARTID. If these are in a class we could use,
	 * reduce the supported partid_max to match the number of intpartid.
	 * If the class is unknown, just ignore it.
	 */
	if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) &&
	    class->type != MPAM_CLASS_UNKNOWN) {
		u16 partid_max;
		u32 nrwidr;

		err = mpam_read_partsel_reg(msc, PARTID_NRW_IDR, &nrwidr);
		if (err)
			return err;

		partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr);

		mpam_set_feature(mpam_feat_partid_nrw, props);
		msc->partid_max = min(msc->partid_max, partid_max);
	}

	return 0;
}

static int mpam_msc_hw_probe(struct mpam_msc *msc)
{
	int ret;
	u64 idr;
	u16 partid_max;
	u8 ris_idx, pmg_max;
	struct mpam_msc_ris *ris;
	struct device *dev = &msc->pdev->dev;

	lockdep_assert_held(&msc->probe_lock);

	if (!mpam_msc_check_aidr(msc)) {
		dev_err_once(dev, "MSC does not match architecture v1.x\n");
		return -EIO;
	}

	/* Grab an IDR value to find out how many RIS there are */
	scoped_guard(mutex, &msc->part_sel_lock) {
		ret = mpam_msc_read_idr(msc, &idr);
		if (ret)
			return ret;

		ret = mpam_read_partsel_reg(msc, IIDR, &msc->iidr);
		if (ret)
			return ret;
	}

	mpam_enable_quirks(msc);

	msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr);

	/* Use these values so partid/pmg always starts with a valid value */
	msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
	msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);

	for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) {
		scoped_guard(mutex, &msc->part_sel_lock) {
			ret = __mpam_part_sel(ris_idx, 0, msc);
			if (ret)
				return ret;

			ret = mpam_msc_read_idr(msc, &idr);
			if (ret)
				return ret;
		}

		partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
		pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
		msc->partid_max = min(msc->partid_max, partid_max);
		msc->pmg_max = min(msc->pmg_max, pmg_max);
		msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr);

		scoped_guard(mutex, &mpam_list_lock) {
			ris = mpam_get_or_create_ris(msc, ris_idx);
			if (IS_ERR(ris))
				return PTR_ERR(ris);
		}
		ris->idr = idr;

		scoped_guard(mutex, &msc->part_sel_lock) {
			ret = __mpam_part_sel(ris_idx, 0, msc);
			if (ret)
				return ret;

			ret = mpam_ris_hw_probe(ris);
			if (ret)
				return ret;
		}
	}

	/* Clear any stale errors */
	ret = mpam_msc_clear_esr(msc);
	if (ret)
		return ret;

	spin_lock(&partid_max_lock);
	mpam_partid_max = min(mpam_partid_max, msc->partid_max);
	mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max);
	spin_unlock(&partid_max_lock);

	msc->probed = true;

	return 0;
}

struct mon_read {
	struct mpam_msc_ris		*ris;
	struct mon_cfg			*ctx;
	enum mpam_device_features	type;
	u64				*val;
	int				err;
	bool				waited_timeout;
};

static int mpam_msc_read_mbwu_l(struct mpam_msc *msc, u64 *res)
{
	int ret;
	int retry = 3;
	u32 mbwu_l_low;
	u32 mbwu_l_high1, mbwu_l_high2;

	mpam_mon_sel_lock_held(msc);

	if (msc->iface == MPAM_IFACE_MMIO) {
		WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
		WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
					       &msc->accessibility));
	}

	ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2);
	if (ret)
		return ret;

	do {
		mbwu_l_high1 = mbwu_l_high2;
		ret = __mpam_read_reg(msc, MSMON_MBWU_L, &mbwu_l_low);
		if (ret)
			return ret;
		ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2);
		if (ret)
			return ret;

		retry--;
	} while (mbwu_l_high1 != mbwu_l_high2 && retry > 0);

	if (mbwu_l_high1 == mbwu_l_high2) {
		*res = ((u64)mbwu_l_high1 << 32) | mbwu_l_low;
	} else {
		pr_warn("Failed to read a stable value\n");
		*res = MSMON___L_NRDY;
	}

	return 0;
}

static int mpam_msc_zero_mbwu_l(struct mpam_msc *msc)
{
	int ret;

	mpam_mon_sel_lock_held(msc);

	if (msc->iface == MPAM_IFACE_MMIO) {
		WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
		WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
					       &msc->accessibility));
	}

	ret = __mpam_write_reg(msc, MSMON_MBWU_L, 0);
	if (ret)
		return ret;

	return __mpam_write_reg(msc, MSMON_MBWU_L + 4, 0);
}

static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
				   u32 *flt_val)
{
	struct mon_cfg *ctx = m->ctx;

	/*
	 * For CSU counters its implementation-defined what happens when not
	 * filtering by partid.
	 */
	*ctl_val = MSMON_CFG_x_CTL_MATCH_PARTID;

	*flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid);

	if (m->ctx->match_pmg) {
		*ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG;
		*flt_val |= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg);
	}

	switch (m->type) {
	case mpam_feat_msmon_csu:
		*ctl_val |= MSMON_CFG_CSU_CTL_TYPE_CSU;

		if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props))
			*flt_val |= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean);

		break;
	case mpam_feat_msmon_mbwu_31counter:
	case mpam_feat_msmon_mbwu_44counter:
	case mpam_feat_msmon_mbwu_63counter:
		*ctl_val |= MSMON_CFG_MBWU_CTL_TYPE_MBWU;

		if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props))
			*flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);

		break;
	default:
		pr_warn("Unexpected monitor type %d\n", m->type);
	}
}

static int read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
				   u32 *flt_val)
{
	struct mpam_msc *msc = m->ris->vmsc->msc;
	int ret;

	switch (m->type) {
	case mpam_feat_msmon_csu:
		ret = mpam_read_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
		if (ret)
			return ret;
		return mpam_read_monsel_reg(msc, CFG_CSU_FLT, flt_val);
	case mpam_feat_msmon_mbwu_31counter:
	case mpam_feat_msmon_mbwu_44counter:
	case mpam_feat_msmon_mbwu_63counter:
		ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
		if (ret)
			return ret;
		return mpam_read_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
	default:
		pr_warn("Unexpected monitor type %d\n", m->type);
		return -EINVAL;
	}
}

/* Remove values set by the hardware to prevent apparent mismatches. */
static inline void clean_msmon_ctl_val(u32 *cur_ctl)
{
	*cur_ctl &= ~MSMON_CFG_x_CTL_OFLOW_STATUS;

	if (FIELD_GET(MSMON_CFG_x_CTL_TYPE, *cur_ctl) == MSMON_CFG_MBWU_CTL_TYPE_MBWU)
		*cur_ctl &= ~MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
}

static int write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val,
				    u32 flt_val)
{
	struct mpam_msc *msc = m->ris->vmsc->msc;
	int ret;

	/*
	 * Write the ctl_val with the enable bit cleared, reset the counter,
	 * then enable counter.
	 */
	switch (m->type) {
	case mpam_feat_msmon_csu:
		ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val);
		if (ret)
			return ret;
		ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
		if (ret)
			return ret;
		ret = mpam_write_monsel_reg(msc, CSU, 0);
		if (ret)
			return ret;
		return mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val | MSMON_CFG_x_CTL_EN);
	case mpam_feat_msmon_mbwu_31counter:
	case mpam_feat_msmon_mbwu_44counter:
	case mpam_feat_msmon_mbwu_63counter:
		ret = mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
		if (ret)
			return ret;
		ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
		if (ret)
			return ret;
		ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL,
					    ctl_val | MSMON_CFG_x_CTL_EN);
		if (ret)
			return ret;
		/* Counting monitors require NRDY to be reset by software */
		if (m->type == mpam_feat_msmon_mbwu_31counter)
			return mpam_write_monsel_reg(msc, MBWU, 0);

		return mpam_msc_zero_mbwu_l(m->ris->vmsc->msc);
	default:
		pr_warn("Unexpected monitor type %d\n", m->type);
		return -EINVAL;
	}
}

static u64 __mpam_msmon_overflow_val(enum mpam_device_features type)
{
	/* TODO: implement scaling counters */
	switch (type) {
	case mpam_feat_msmon_mbwu_63counter:
		return BIT_ULL(hweight_long(MSMON___LWD_VALUE));
	case mpam_feat_msmon_mbwu_44counter:
		return BIT_ULL(hweight_long(MSMON___L_VALUE));
	case mpam_feat_msmon_mbwu_31counter:
		return BIT_ULL(hweight_long(MSMON___VALUE));
	default:
		return 0;
	}
}

static u64 mpam_msmon_overflow_val(enum mpam_device_features type,
				   struct mpam_msc *msc)
{
	u64 overflow_val = __mpam_msmon_overflow_val(type);

	if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc))
		overflow_val *= 64;

	return overflow_val;
}

/*
 * This function might be called via smp_call_function_any(), so propagate
 * errors inside the arg struct.
 */
static void __ris_msmon_read_locked(struct mon_read *m)
{
	u64 now;
	u32 now32;
	bool nrdy = false;
	bool config_mismatch;
	bool overflow = false;
	struct mon_cfg *ctx = m->ctx;
	bool reset_on_next_read = false;
	struct mpam_msc_ris *ris = m->ris;
	struct msmon_mbwu_state *mbwu_state;
	struct mpam_msc *msc = m->ris->vmsc->msc;
	u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;

	mpam_mon_sel_lock_held(msc);

	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) |
		  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
	m->err = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
	if (m->err)
		return;

	switch (m->type) {
	case mpam_feat_msmon_mbwu_31counter:
	case mpam_feat_msmon_mbwu_44counter:
	case mpam_feat_msmon_mbwu_63counter:
		mbwu_state = &ris->mbwu_state[ctx->mon];
		if (mbwu_state) {
			reset_on_next_read = mbwu_state->reset_on_next_read;
			mbwu_state->reset_on_next_read = false;
		}
		break;
	default:
		break;
	}

	/*
	 * Read the existing configuration to avoid re-writing the same values.
	 * This saves waiting for 'nrdy' on subsequent reads.
	 */
	m->err = read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt);
	if (m->err)
		return;

	if (mpam_feat_msmon_mbwu_31counter == m->type)
		overflow = cur_ctl & MSMON_CFG_x_CTL_OFLOW_STATUS;
	else if (mpam_feat_msmon_mbwu_44counter == m->type ||
		 mpam_feat_msmon_mbwu_63counter == m->type)
		overflow = cur_ctl & MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;

	clean_msmon_ctl_val(&cur_ctl);
	gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val);
	config_mismatch = cur_flt != flt_val ||
			  cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN);

	if (config_mismatch || reset_on_next_read) {
		m->err = write_msmon_ctl_flt_vals(m, ctl_val, flt_val);
		if (m->err)
			return;
		overflow = false;
	} else if (overflow) {
		m->err = mpam_write_monsel_reg(msc, CFG_MBWU_CTL,
					       cur_ctl &
					       ~(MSMON_CFG_x_CTL_OFLOW_STATUS |
					       MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L));
		if (m->err)
			return;
	}

	switch (m->type) {
	case mpam_feat_msmon_csu:
		m->err = mpam_read_monsel_reg(msc, CSU, &now32);
		if (m->err)
			return;
		nrdy = now32 & MSMON___NRDY;
		now = FIELD_GET(MSMON___VALUE, now32);

		if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout)
			nrdy = false;

		break;
	case mpam_feat_msmon_mbwu_31counter:
	case mpam_feat_msmon_mbwu_44counter:
	case mpam_feat_msmon_mbwu_63counter:
		if (m->type != mpam_feat_msmon_mbwu_31counter) {
			m->err = mpam_msc_read_mbwu_l(msc, &now);
			if (m->err)
				return;
			nrdy = now & MSMON___L_NRDY;

			if (m->type == mpam_feat_msmon_mbwu_63counter)
				now = FIELD_GET(MSMON___LWD_VALUE, now);
			else
				now = FIELD_GET(MSMON___L_VALUE, now);
		} else {
			m->err = mpam_read_monsel_reg(msc, MBWU, &now32);
			if (m->err)
				return;
			nrdy = now32 & MSMON___NRDY;
			now = FIELD_GET(MSMON___VALUE, now32);
		}

		if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc))
			now *= 64;

		if (nrdy)
			break;

		mbwu_state = &ris->mbwu_state[ctx->mon];

		if (overflow)
			mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc);

		/*
		 * Include bandwidth consumed before the last hardware reset and
		 * a counter size increment for each overflow.
		 */
		now += mbwu_state->correction;
		break;
	default:
		m->err = -EINVAL;
	}

	if (nrdy)
		m->err = -EBUSY;

	if (m->err)
		return;

	*m->val += now;
}

static void __ris_msmon_read(void *arg)
{
	struct mon_read *m = arg;
	struct mpam_msc *msc = m->ris->vmsc->msc;

	if (!mpam_mon_sel_lock(msc)) {
		m->err = -EIO;
		return;
	}

	__ris_msmon_read_locked(m);

	mpam_mon_sel_unlock(msc);
}

static int _msmon_read(struct mpam_component *comp, struct mon_read *arg)
{
	int err, any_err = 0;
	struct mpam_vmsc *vmsc;

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		struct mpam_msc *msc = vmsc->msc;
		struct mpam_msc_ris *ris;

		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
					 srcu_read_lock_held(&mpam_srcu)) {
			arg->ris = ris;

			if (msc->iface == MPAM_IFACE_MMIO) {
				err = smp_call_function_any(&msc->accessibility,
							    __ris_msmon_read,
							    arg, true);
				if (!err)
					err = arg->err;
			} else {
				__ris_msmon_read(arg);
				err = arg->err;
			}

			/*
			 * Save one error to be returned to the caller, but
			 * keep reading counters so that get reprogrammed. On
			 * platforms with NRDY this lets us wait once.
			 */
			if (err)
				any_err = err;
		}
	}

	return any_err;
}

static enum mpam_device_features mpam_msmon_choose_counter(struct mpam_class *class)
{
	struct mpam_props *cprops = &class->props;

	if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops))
		return mpam_feat_msmon_mbwu_63counter;
	if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, cprops))
		return mpam_feat_msmon_mbwu_44counter;

	return mpam_feat_msmon_mbwu_31counter;
}

int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx,
		    enum mpam_device_features type, u64 *val)
{
	int err;
	struct mon_read arg;
	u64 wait_jiffies = 0;
	struct mpam_class *class = comp->class;
	struct mpam_props *cprops = &class->props;

	might_sleep();

	if (!mpam_is_enabled())
		return -EIO;

	if (!mpam_has_feature(type, cprops))
		return -EOPNOTSUPP;

	if (type == mpam_feat_msmon_mbwu)
		type = mpam_msmon_choose_counter(class);

	arg = (struct mon_read) {
		.ctx = ctx,
		.type = type,
		.val = val,
	};
	*val = 0;

	err = _msmon_read(comp, &arg);
	if (err == -EBUSY && class->nrdy_usec)
		wait_jiffies = usecs_to_jiffies(class->nrdy_usec);

	while (wait_jiffies)
		wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies);

	if (err == -EBUSY) {
		arg = (struct mon_read) {
			.ctx = ctx,
			.type = type,
			.val = val,
			.waited_timeout = true,
		};
		*val = 0;

		err = _msmon_read(comp, &arg);
	}

	return err;
}

void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx)
{
	struct mpam_msc *msc;
	struct mpam_vmsc *vmsc;
	struct mpam_msc_ris *ris;

	if (!mpam_is_enabled())
		return;

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props))
			continue;

		msc = vmsc->msc;
		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
					 srcu_read_lock_held(&mpam_srcu)) {
			if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
				continue;

			if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
				continue;

			ris->mbwu_state[ctx->mon].correction = 0;
			ris->mbwu_state[ctx->mon].reset_on_next_read = true;
			mpam_mon_sel_unlock(msc);
		}
	}
}

static int mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd)
{
	u32 num_words, msb;
	u32 bm = ~0;
	int i;

	lockdep_assert_held(&msc->part_sel_lock);

	if (wd == 0)
		return 0;

	/*
	 * Write all ~0 to all but the last 32bit-word, which may
	 * have fewer bits...
	 */
	num_words = DIV_ROUND_UP(wd, 32);
	for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) {
		int ret;

		ret = __mpam_write_reg(msc, reg, bm);
		if (ret)
			return ret;
	}

	/*
	 * ....and then the last (maybe) partial 32bit word. When wd is a
	 * multiple of 32, msb should be 31 to write a full 32bit word.
	 */
	msb = (wd - 1) % 32;
	bm = GENMASK(msb, 0);
	return __mpam_write_reg(msc, reg, bm);
}

static void mpam_apply_t241_erratum(struct mpam_msc_ris *ris, u16 partid)
{
	int sidx, i, lcount = 1000;
	void __iomem *regs;
	u64 val0, val;

	regs = t241_scratch_regs[ris->vmsc->msc->t241_id];

	for (i = 0; i < lcount; i++) {
		/* Read the shadow register at index 0 */
		val0 = readq_relaxed(regs + T241_SHADOW_REG_OFF(0, partid));

		/* Check if all the shadow registers have the same value */
		for (sidx = 1; sidx < T241_CHIP_NSLICES; sidx++) {
			val = readq_relaxed(regs +
					    T241_SHADOW_REG_OFF(sidx, partid));
			if (val != val0)
				break;
		}
		if (sidx == T241_CHIP_NSLICES)
			break;
	}

	if (i == lcount)
		pr_warn_once("t241: inconsistent values in shadow regs");

	/* Write a value zero to spare registers to take effect of MBW conf */
	writeq_relaxed(0, regs + T241_SPARE_REG0_OFF);
	writeq_relaxed(0, regs + T241_SPARE_REG1_OFF);
}

static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid,
					  struct mpam_config *cfg)
{
	if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc))
		mpam_apply_t241_erratum(ris, partid);
}

static u16 mpam_wa_t241_force_mbw_min_to_one(struct mpam_props *props)
{
	u16 max_hw_value, min_hw_granule, res0_bits;

	res0_bits = 16 - props->bwa_wd;
	max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits;
	min_hw_granule = ~max_hw_value;

	return min_hw_granule + 1;
}

static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props,
					  struct mpam_config *cfg)
{
	u16 val = 0;
	u16 max;
	u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1;

	if (mpam_has_feature(mpam_feat_mbw_max, cfg)) {
		max = cfg->mbw_max;
	} else {
		/* Resetting. Hence, use the ris specific default. */
		max = GENMASK(15, 16 - props->bwa_wd);
	}

	if (max > delta)
		val = max - delta;

	return val;
}

/* Called via IPI. Call while holding an SRCU reference */
static int mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
				     struct mpam_config *cfg)
{
	u16 cmax = MPAMCFG_CMAX_CMAX;
	struct mpam_msc *msc = ris->vmsc->msc;
	struct mpam_props *rprops = &ris->props;
	int ret;

	guard(mutex)(&msc->part_sel_lock);
	ret = __mpam_part_sel(ris->ris_idx, partid, msc);
	if (ret)
		return ret;

	if (mpam_has_feature(mpam_feat_partid_nrw, rprops)) {
		/* Update the intpartid mapping */
		ret = mpam_write_partsel_reg(msc, INTPARTID,
					     MPAMCFG_INTPARTID_INTERNAL | partid);
		if (ret)
			return ret;

		/*
		 * Then switch to the 'internal' partid to update the
		 * configuration.
		 */
		ret = __mpam_intpart_sel(ris->ris_idx, partid, msc);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_cpor_part, rprops)) {
		if (mpam_has_feature(mpam_feat_cpor_part, cfg)) {
			ret = mpam_write_partsel_reg(msc, CPBM, cfg->cpbm);
			if (ret)
				return ret;
		} else {
			ret = mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM,
						    rprops->cpbm_wd);
			if (ret)
				return ret;
		}
	}

	if (mpam_has_feature(mpam_feat_mbw_part, rprops)) {
		if (mpam_has_feature(mpam_feat_mbw_part, cfg)) {
			ret = mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm);
			if (ret)
				return ret;
		} else {
			ret = mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM,
						    rprops->mbw_pbm_bits);
			if (ret)
				return ret;
		}
	}

	if (mpam_has_feature(mpam_feat_mbw_min, rprops)) {
		u16 val = 0;

		if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, msc)) {
			u16 min = mpam_wa_t241_force_mbw_min_to_one(rprops);

			val = mpam_wa_t241_calc_min_from_max(rprops, cfg);
			val = max(val, min);
		}

		ret = mpam_write_partsel_reg(msc, MBW_MIN, val);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_mbw_max, rprops)) {
		if (mpam_has_feature(mpam_feat_mbw_max, cfg)) {
			ret = mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max);
			if (ret)
				return ret;
		} else {
			ret = mpam_write_partsel_reg(msc, MBW_MAX,
						     MPAMCFG_MBW_MAX_MAX);
			if (ret)
				return ret;
		}
	}

	if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) {
		ret = mpam_write_partsel_reg(msc, MBW_PROP, 0);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) {
		ret = mpam_write_partsel_reg(msc, CMAX, cmax);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) {
		ret = mpam_write_partsel_reg(msc, CMIN, 0);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) {
		ret = mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC);
		if (ret)
			return ret;
	}

	if (mpam_has_feature(mpam_feat_intpri_part, rprops) ||
	    mpam_has_feature(mpam_feat_dspri_part, rprops)) {
		u32 pri_val = 0;

		if (mpam_has_feature(mpam_feat_intpri_part, rprops)) {
			u16 intpri = GENMASK(rprops->intpri_wd - 1, 0);

			/* aces high? */
			if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
				intpri = 0;

			pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri);
		}
		if (mpam_has_feature(mpam_feat_dspri_part, rprops)) {
			u16 dspri = GENMASK(rprops->dspri_wd - 1, 0);

			if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
				dspri = 0;

			pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri);
		}

		ret = mpam_write_partsel_reg(msc, PRI, pri_val);
		if (ret)
			return ret;
	}

	mpam_quirk_post_config_change(ris, partid, cfg);

	return 0;
}

/* Call with msc cfg_lock held */
static int mpam_restore_mbwu_state(void *_ris)
{
	int i;
	u64 val;
	struct mon_read mbwu_arg;
	struct mpam_msc_ris *ris = _ris;
	struct msmon_mbwu_state *mbwu_state;
	struct mpam_msc *msc = ris->vmsc->msc;
	struct mpam_class *class = ris->vmsc->comp->class;

	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
		if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
			return -EIO;

		mbwu_state = &ris->mbwu_state[i];

		if (!mbwu_state->enabled) {
			mpam_mon_sel_unlock(msc);
			continue;
		}

		val = 0;
		mbwu_arg = (struct mon_read) {
			.ris = ris,
			.ctx = &mbwu_state->cfg,
			.type = mpam_msmon_choose_counter(class),
			.val = &val,
		};

		mbwu_state->reset_on_next_read = true;

		__ris_msmon_read_locked(&mbwu_arg);

		mpam_mon_sel_unlock(msc);

		if (mbwu_arg.err && mbwu_arg.err != -EBUSY)
			return mbwu_arg.err;
	}

	return 0;
}

/* Call with MSC cfg_lock held */
static int mpam_save_mbwu_state(void *arg)
{
	int i;
	int ret;
	u64 val;
	struct mon_cfg *cfg;
	struct mon_read mbwu_arg;
	u32 cur_flt, cur_ctl, mon_sel;
	struct mpam_msc_ris *ris = arg;
	struct msmon_mbwu_state *mbwu_state;
	struct mpam_msc *msc = ris->vmsc->msc;
	struct mpam_class *class = ris->vmsc->comp->class;

	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
		if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
			return -EIO;

		mbwu_state = &ris->mbwu_state[i];
		cfg = &mbwu_state->cfg;

		mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) |
			  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
		ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
		if (ret)
			return ret;
		ret = mpam_read_monsel_reg(msc, CFG_MBWU_FLT, &cur_flt);
		if (ret)
			return ret;
		ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, &cur_ctl);
		if (ret)
			return ret;

		cfg->mon = i;
		cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt);
		cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl);
		cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt);
		mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl);

		if (!mbwu_state->enabled) {
			mpam_mon_sel_unlock(msc);
			continue;
		}

		val = 0;
		mbwu_arg = (struct mon_read) {
			.ris = ris,
			.ctx = cfg,
			.type = mpam_msmon_choose_counter(class),
			.val = &val,
		};

		__ris_msmon_read_locked(&mbwu_arg);

		mbwu_state->reset_on_next_read = true;
		if (!mbwu_arg.err) {
			/*
			 * __ris_msmon_read_locked() already included the
			 * previous correction value.
			 */
			mbwu_state->correction = val;
		}

		ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0);
		if (ret)
			return ret;

		mpam_mon_sel_unlock(msc);
	}

	return 0;
}

/*
 * Called via smp_call_on_cpu() to prevent migration, while still being
 * pre-emptible. Caller must hold mpam_srcu.
 */
static int mpam_reset_ris(void *arg)
{
	u16 partid_max;
	struct mpam_config reset_cfg = {};
	struct mpam_msc_ris *ris = arg;
	int ret;

	if (ris->in_reset_state)
		return 0;

	spin_lock(&partid_max_lock);
	partid_max = mpam_partid_max;
	spin_unlock(&partid_max_lock);
	for (u32 partid = 0; partid <= partid_max; partid++) {
		ret = mpam_reprogram_ris_partid(ris, partid, &reset_cfg);
		if (ret)
			return ret;
	}

	return 0;
}

/*
 * Get the preferred CPU for this MSC. If it is accessible from this CPU,
 * this CPU is preferred. This can be preempted/migrated, it will only result
 * in more work.
 */
static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc)
{
	int cpu = raw_smp_processor_id();

	if (cpumask_test_cpu(cpu, &msc->accessibility))
		return cpu;

	return cpumask_first_and(&msc->accessibility, cpu_online_mask);
}

static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg)
{
	if (msc->iface != MPAM_IFACE_MMIO)
		return fn(arg);

	lockdep_assert_irqs_enabled();
	lockdep_assert_cpus_held();
	WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu)));

	return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true);
}

struct mpam_write_config_arg {
	struct mpam_msc_ris *ris;
	struct mpam_component *comp;
	u16 partid;
};

static int __write_config(void *arg)
{
	struct mpam_write_config_arg *c = arg;

	return mpam_reprogram_ris_partid(c->ris, c->partid,
					 &c->comp->cfg[c->partid]);
}

static void mpam_reprogram_msc(struct mpam_msc *msc)
{
	bool reset;
	struct mpam_config *cfg;
	struct mpam_msc_ris *ris;
	struct mpam_write_config_arg arg;

	/*
	 * No lock for mpam_partid_max as partid_max_published has been
	 * set by mpam_enabled(), so the values can no longer change.
	 */
	mpam_assert_partid_sizes_fixed();

	mutex_lock(&msc->cfg_lock);
	list_for_each_entry_srcu(ris, &msc->ris, msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		if (!mpam_is_enabled() && !ris->in_reset_state) {
			mpam_touch_msc(msc, &mpam_reset_ris, ris);
			ris->in_reset_state = true;
			continue;
		}

		arg.comp = ris->vmsc->comp;
		arg.ris = ris;
		reset = true;
		for (u32 partid = 0; partid <= mpam_partid_max; partid++) {
			cfg = &ris->vmsc->comp->cfg[partid];
			if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST))
				reset = false;

			arg.partid = partid;
			mpam_touch_msc(msc, __write_config, &arg);
		}
		ris->in_reset_state = reset;

		if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
			mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris);
	}
	mutex_unlock(&msc->cfg_lock);
}

static int mpam_enable_msc_ecr(void *_msc)
{
	struct mpam_msc *msc = _msc;

	return __mpam_write_reg(msc, MPAMF_ECR, MPAMF_ECR_INTEN);
}

/* This can run in mpam_disable(), and the interrupt handler on the same CPU */
static int mpam_disable_msc_ecr(void *_msc)
{
	struct mpam_msc *msc = _msc;

	return __mpam_write_reg(msc, MPAMF_ECR, 0);
}

static void _enable_percpu_irq(void *_irq)
{
	int *irq = _irq;

	enable_percpu_irq(*irq, IRQ_TYPE_NONE);
}

static int mpam_cpu_online(unsigned int cpu)
{
	struct mpam_msc *msc;

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		if (!cpumask_test_cpu(cpu, &msc->accessibility))
			continue;

		if (msc->reenable_error_ppi)
			_enable_percpu_irq(&msc->reenable_error_ppi);

		if (atomic_fetch_inc(&msc->online_refs) == 0) {
			mutex_lock(&msc->error_irq_lock);
			if (msc->error_irq_hw_enabled)
				mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
			mutex_unlock(&msc->error_irq_lock);

			mpam_reprogram_msc(msc);
		}
	}

	if (mpam_resctrl_enabled)
		return mpam_resctrl_online_cpu(cpu);

	return 0;
}

/* Before mpam is enabled, try to probe new MSC */
static int mpam_discovery_cpu_online(unsigned int cpu)
{
	int err = 0;
	struct mpam_msc *msc;
	bool new_device_probed = false;

	if (mpam_is_enabled())
		return 0;

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		if (!cpumask_test_cpu(cpu, &msc->accessibility))
			continue;

		mutex_lock(&msc->probe_lock);
		if (!msc->probed)
			err = mpam_msc_hw_probe(msc);
		mutex_unlock(&msc->probe_lock);

		if (err)
			break;
		new_device_probed = true;
	}

	if (new_device_probed && !err)
		schedule_work(&mpam_enable_work);
	if (err) {
		mpam_disable_reason = "error during probing";
		schedule_work(&mpam_broken_work);
	}

	return err;
}

static int mpam_cpu_offline(unsigned int cpu)
{
	struct mpam_msc *msc;

	if (mpam_resctrl_enabled)
		mpam_resctrl_offline_cpu(cpu);

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		if (!cpumask_test_cpu(cpu, &msc->accessibility))
			continue;

		if (msc->reenable_error_ppi)
			disable_percpu_irq(msc->reenable_error_ppi);

		if (atomic_dec_and_test(&msc->online_refs)) {
			struct mpam_msc_ris *ris;

			mutex_lock(&msc->cfg_lock);
			list_for_each_entry_srcu(ris, &msc->ris, msc_list,
						 srcu_read_lock_held(&mpam_srcu)) {
				mpam_touch_msc(msc, &mpam_reset_ris, ris);

				/*
				 * The reset state for non-zero partid may be
				 * lost while the CPUs are offline.
				 */
				ris->in_reset_state = false;

				if (mpam_is_enabled())
					mpam_touch_msc(msc, &mpam_save_mbwu_state, ris);
			}
			mutex_unlock(&msc->cfg_lock);
		}
	}

	return 0;
}

static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online),
					  int (*offline)(unsigned int offline),
					  char *name)
{
	mutex_lock(&mpam_cpuhp_state_lock);
	if (mpam_cpuhp_state) {
		cpuhp_remove_state(mpam_cpuhp_state);
		mpam_cpuhp_state = 0;
	}

	mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, name, online,
					     offline);
	if (mpam_cpuhp_state <= 0) {
		pr_err("Failed to register cpuhp callbacks");
		mpam_cpuhp_state = 0;
	}
	mutex_unlock(&mpam_cpuhp_state_lock);
}

static int __setup_ppi(struct mpam_msc *msc)
{
	int cpu;

	msc->error_dev_id = alloc_percpu(struct mpam_msc *);
	if (!msc->error_dev_id)
		return -ENOMEM;

	for_each_cpu(cpu, &msc->accessibility)
		*per_cpu_ptr(msc->error_dev_id, cpu) = msc;

	return 0;
}

static int mpam_msc_setup_error_irq(struct mpam_msc *msc)
{
	int irq;

	irq = platform_get_irq_byname_optional(msc->pdev, "error");
	if (irq <= 0)
		return 0;

	/* Allocate and initialise the percpu device pointer for PPI */
	if (irq_is_percpu(irq))
		return __setup_ppi(msc);

	/* sanity check: shared interrupts can be routed anywhere? */
	if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) {
		pr_err_once("msc:%u is a private resource with a shared error interrupt",
			    msc->id);
		return -EINVAL;
	}

	return 0;
}

/*
 * An MSC can control traffic from a set of CPUs, but may only be accessible
 * from a (hopefully wider) set of CPUs. The common reason for this is power
 * management. If all the CPUs in a cluster are in PSCI:CPU_SUSPEND, the
 * corresponding cache may also be powered off. By making accesses from
 * one of those CPUs, we ensure we don't access a cache that's powered off.
 */
static void update_msc_accessibility(struct mpam_msc *msc)
{
	u32 affinity_id;
	int err;

	err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity",
				       &affinity_id);
	if (err)
		cpumask_copy(&msc->accessibility, cpu_possible_mask);
	else
		acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility);
}

/*
 * There are two ways of reaching a struct mpam_msc_ris. Via the
 * class->component->vmsc->ris, or via the msc.
 * When destroying the msc, the other side needs unlinking and cleaning up too.
 */
static void mpam_msc_destroy(struct mpam_msc *msc)
{
	struct platform_device *pdev = msc->pdev;
	struct mpam_msc_ris *ris, *tmp;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list)
		mpam_ris_destroy(ris);

	list_del_rcu(&msc->all_msc_list);
	synchronize_srcu(&mpam_srcu);
	mpam_pcc_chan_put(msc->pcc_chan);
	platform_set_drvdata(pdev, NULL);

	add_to_garbage(msc);
}

static void mpam_msc_drv_remove(struct platform_device *pdev)
{
	struct mpam_msc *msc = platform_get_drvdata(pdev);

	if (!msc)
		return;

	mutex_lock(&mpam_list_lock);
	mpam_msc_destroy(msc);
	mutex_unlock(&mpam_list_lock);

	mpam_free_garbage();
}

static struct mpam_msc *do_mpam_msc_drv_probe(struct platform_device *pdev)
{
	int err;
	u32 pcc_subspace_id;
	struct mpam_msc *msc;
	struct resource *msc_res;
	struct device *dev = &pdev->dev;

	lockdep_assert_held(&mpam_list_lock);

	msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL);
	if (!msc)
		return ERR_PTR(-ENOMEM);
	init_garbage(&msc->garbage);
	msc->garbage.pdev = pdev;

	err = devm_mutex_init(dev, &msc->probe_lock);
	if (err)
		return ERR_PTR(err);

	err = devm_mutex_init(dev, &msc->part_sel_lock);
	if (err)
		return ERR_PTR(err);

	err = devm_mutex_init(dev, &msc->error_irq_lock);
	if (err)
		return ERR_PTR(err);

	err = devm_mutex_init(dev, &msc->cfg_lock);
	if (err)
		return ERR_PTR(err);

	msc->id = pdev->id;
	msc->pdev = pdev;
	INIT_LIST_HEAD_RCU(&msc->all_msc_list);
	INIT_LIST_HEAD_RCU(&msc->ris);

	update_msc_accessibility(msc);
	if (cpumask_empty(&msc->accessibility)) {
		dev_err_once(dev, "MSC is not accessible from any CPU!");
		return ERR_PTR(-EINVAL);
	}

	err = mpam_msc_setup_error_irq(msc);
	if (err)
		return ERR_PTR(err);

	if (device_property_read_u32(dev, "pcc-channel", &pcc_subspace_id))
		msc->iface = MPAM_IFACE_MMIO;
	else
		msc->iface = MPAM_IFACE_PCC;

	/* Lock type depends on MSC interface used */
	err = mpam_mon_sel_lock_init(dev, msc);
	if (err)
		return ERR_PTR(err);

	if (msc->iface == MPAM_IFACE_MMIO) {
		void __iomem *io;

		io = devm_platform_get_and_ioremap_resource(pdev, 0,
							    &msc_res);
		if (IS_ERR(io)) {
			dev_err_once(dev, "Failed to map MSC base address\n");
			return ERR_CAST(io);
		}
		msc->mapped_hwpage_sz = msc_res->end - msc_res->start;
		msc->mapped_hwpage = io;
	} else if (msc->iface == MPAM_IFACE_PCC) {
		u32 mpam_fb_msc_id;

		msc->pcc_chan = mpam_pcc_chan_get(dev, pcc_subspace_id);
		if (IS_ERR(msc->pcc_chan)) {
			pr_err("Failed to request MSC PCC channel\n");
			return ERR_CAST(msc->pcc_chan);
		}

		err = mpam_fb_check_shared_buffer_size(msc);
		if (err) {
			mpam_pcc_chan_put(msc->pcc_chan);

			return ERR_PTR(err);
		}

		err = mpam_fb_check_protocol_version(msc);
		if (err) {
			mpam_pcc_chan_put(msc->pcc_chan);

			return ERR_PTR(err);
		}

		if (device_property_read_u32(&pdev->dev, "mpam-fb-msc-id",
					     &mpam_fb_msc_id))
			mpam_fb_msc_id = msc->id;
		msc->fb_id = mpam_fb_msc_id;
	} else {
		return ERR_PTR(-EINVAL);
	}

	list_add_rcu(&msc->all_msc_list, &mpam_all_msc);
	platform_set_drvdata(pdev, msc);

	return msc;
}

static int fw_num_msc;

static int mpam_msc_drv_probe(struct platform_device *pdev)
{
	int err;
	struct mpam_msc *msc = NULL;
	void *plat_data = pdev->dev.platform_data;

	mutex_lock(&mpam_list_lock);
	msc = do_mpam_msc_drv_probe(pdev);
	mutex_unlock(&mpam_list_lock);

	if (IS_ERR(msc))
		return PTR_ERR(msc);

	/* Create RIS entries described by firmware */
	err = acpi_mpam_parse_resources(msc, plat_data);
	if (err) {
		mpam_msc_drv_remove(pdev);
		return err;
	}

	if (atomic_add_return(1, &mpam_num_msc) == fw_num_msc)
		mpam_register_cpuhp_callbacks(mpam_discovery_cpu_online, NULL,
					      "mpam:drv_probe");

	return 0;
}

static struct platform_driver mpam_msc_driver = {
	.driver = {
		.name = "mpam_msc",
		.suppress_bind_attrs = true,
	},
	.probe = mpam_msc_drv_probe,
	.remove = mpam_msc_drv_remove,
};

/* Any of these features mean the BWA_WD field is valid. */
static bool mpam_has_bwa_wd_feature(struct mpam_props *props)
{
	if (mpam_has_feature(mpam_feat_mbw_min, props))
		return true;
	if (mpam_has_feature(mpam_feat_mbw_max, props))
		return true;
	if (mpam_has_feature(mpam_feat_mbw_prop, props))
		return true;
	return false;
}

/* Any of these features mean the CMAX_WD field is valid. */
static bool mpam_has_cmax_wd_feature(struct mpam_props *props)
{
	if (mpam_has_feature(mpam_feat_cmax_cmax, props))
		return true;
	if (mpam_has_feature(mpam_feat_cmax_cmin, props))
		return true;
	return false;
}

#define MISMATCHED_HELPER(parent, child, helper, field, alias)		\
	helper(parent) &&						\
	((helper(child) && (parent)->field != (child)->field) ||	\
	 (!helper(child) && !(alias)))

#define MISMATCHED_FEAT(parent, child, feat, field, alias)		     \
	mpam_has_feature((feat), (parent)) &&				     \
	((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) || \
	 (!mpam_has_feature((feat), (child)) && !(alias)))

#define CAN_MERGE_FEAT(parent, child, feat, alias)			\
	(alias) && !mpam_has_feature((feat), (parent)) &&		\
	mpam_has_feature((feat), (child))

/*
 * Combine two props fields.
 * If this is for controls that alias the same resource, it is safe to just
 * copy the values over. If two aliasing controls implement the same scheme
 * a safe value must be picked.
 * For non-aliasing controls, these control different resources, and the
 * resulting safe value must be compatible with both. When merging values in
 * the tree, all the aliasing resources must be handled first.
 * On mismatch, parent is modified.
 * Quirks on an MSC will apply to all MSC in that class.
 */
static void __props_mismatch(struct mpam_props *parent,
			     struct mpam_props *child, bool alias)
{
	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cpor_part, alias)) {
		parent->cpbm_wd = child->cpbm_wd;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cpor_part,
				   cpbm_wd, alias)) {
		pr_debug("cleared cpor_part\n");
		mpam_clear_feature(mpam_feat_cpor_part, parent);
		parent->cpbm_wd = 0;
	}

	if (CAN_MERGE_FEAT(parent, child, mpam_feat_mbw_part, alias)) {
		parent->mbw_pbm_bits = child->mbw_pbm_bits;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_mbw_part,
				   mbw_pbm_bits, alias)) {
		pr_debug("cleared mbw_part\n");
		mpam_clear_feature(mpam_feat_mbw_part, parent);
		parent->mbw_pbm_bits = 0;
	}

	/* bwa_wd is a count of bits, fewer bits means less precision */
	if (alias && !mpam_has_bwa_wd_feature(parent) &&
	    mpam_has_bwa_wd_feature(child)) {
		parent->bwa_wd = child->bwa_wd;
	} else if (MISMATCHED_HELPER(parent, child, mpam_has_bwa_wd_feature,
				     bwa_wd, alias)) {
		pr_debug("took the min bwa_wd\n");
		parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd);
	}

	if (alias && !mpam_has_cmax_wd_feature(parent) && mpam_has_cmax_wd_feature(child)) {
		parent->cmax_wd = child->cmax_wd;
	} else if (MISMATCHED_HELPER(parent, child, mpam_has_cmax_wd_feature,
				     cmax_wd, alias)) {
		pr_debug("%s took the min cmax_wd\n", __func__);
		parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd);
	}

	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cmax_cassoc, alias)) {
		parent->cassoc_wd = child->cassoc_wd;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cmax_cassoc,
				   cassoc_wd, alias)) {
		pr_debug("%s cleared cassoc_wd\n", __func__);
		mpam_clear_feature(mpam_feat_cmax_cassoc, parent);
		parent->cassoc_wd = 0;
	}

	/* For num properties, take the minimum */
	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_csu, alias)) {
		parent->num_csu_mon = child->num_csu_mon;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_csu,
				   num_csu_mon, alias)) {
		pr_debug("took the min num_csu_mon\n");
		parent->num_csu_mon = min(parent->num_csu_mon,
					  child->num_csu_mon);
	}

	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_mbwu, alias)) {
		parent->num_mbwu_mon = child->num_mbwu_mon;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_mbwu,
				   num_mbwu_mon, alias)) {
		pr_debug("took the min num_mbwu_mon\n");
		parent->num_mbwu_mon = min(parent->num_mbwu_mon,
					   child->num_mbwu_mon);
	}

	if (CAN_MERGE_FEAT(parent, child, mpam_feat_intpri_part, alias)) {
		parent->intpri_wd = child->intpri_wd;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_intpri_part,
				   intpri_wd, alias)) {
		pr_debug("%s took the min intpri_wd\n", __func__);
		parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd);
	}

	if (CAN_MERGE_FEAT(parent, child, mpam_feat_dspri_part, alias)) {
		parent->dspri_wd = child->dspri_wd;
	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_dspri_part,
				   dspri_wd, alias)) {
		pr_debug("%s took the min dspri_wd\n", __func__);
		parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd);
	}

	/* TODO: alias support for these two */
	/* {int,ds}pri may not have differing 0-low behaviour */
	if (mpam_has_feature(mpam_feat_intpri_part, parent) &&
	    (!mpam_has_feature(mpam_feat_intpri_part, child) ||
	     mpam_has_feature(mpam_feat_intpri_part_0_low, parent) !=
	     mpam_has_feature(mpam_feat_intpri_part_0_low, child))) {
		pr_debug("%s cleared intpri_part\n", __func__);
		mpam_clear_feature(mpam_feat_intpri_part, parent);
		mpam_clear_feature(mpam_feat_intpri_part_0_low, parent);
	}
	if (mpam_has_feature(mpam_feat_dspri_part, parent) &&
	    (!mpam_has_feature(mpam_feat_dspri_part, child) ||
	     mpam_has_feature(mpam_feat_dspri_part_0_low, parent) !=
	     mpam_has_feature(mpam_feat_dspri_part_0_low, child))) {
		pr_debug("%s cleared dspri_part\n", __func__);
		mpam_clear_feature(mpam_feat_dspri_part, parent);
		mpam_clear_feature(mpam_feat_dspri_part_0_low, parent);
	}

	if (alias) {
		/* Merge features for aliased resources */
		bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
	} else {
		/* Clear missing features for non aliasing */
		bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
	}
}

/*
 * If a vmsc doesn't match class feature/configuration, do the right thing(tm).
 * For 'num' properties we can just take the minimum.
 * For properties where the mismatched unused bits would make a difference, we
 * nobble the class feature, as we can't configure all the resources.
 * e.g. The L3 cache is composed of two resources with 13 and 17 portion
 * bitmaps respectively.
 * Quirks on an MSC will apply to all MSC in that class.
 */
static void
__class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc)
{
	struct mpam_props *cprops = &class->props;
	struct mpam_props *vprops = &vmsc->props;
	struct device *dev = &vmsc->msc->pdev->dev;

	lockdep_assert_held(&mpam_list_lock); /* we modify class */

	dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n",
		(long)cprops->features, (long)vprops->features);

	/* Merge quirks */
	class->quirks |= vmsc->msc->quirks;

	/* Take the safe value for any common features */
	__props_mismatch(cprops, vprops, false);
}

static void
__vmsc_props_mismatch(struct mpam_vmsc *vmsc, struct mpam_msc_ris *ris)
{
	struct mpam_props *rprops = &ris->props;
	struct mpam_props *vprops = &vmsc->props;
	struct device *dev = &vmsc->msc->pdev->dev;

	lockdep_assert_held(&mpam_list_lock); /* we modify vmsc */

	dev_dbg(dev, "Merging features for vmsc:0x%lx |= ris:0x%lx\n",
		(long)vprops->features, (long)rprops->features);

	/*
	 * Merge mismatched features - Copy any features that aren't common,
	 * but take the safe value for any common features.
	 */
	__props_mismatch(vprops, rprops, true);
}

/*
 * Copy the first component's first vMSC's properties and features to the
 * class. __class_props_mismatch() will remove conflicts.
 * It is not possible to have a class with no components, or a component with
 * no resources. The vMSC properties have already been built.
 */
static void mpam_enable_init_class_features(struct mpam_class *class)
{
	struct mpam_vmsc *vmsc;
	struct mpam_component *comp;

	comp = list_first_entry(&class->components,
				struct mpam_component, class_list);
	vmsc = list_first_entry(&comp->vmsc,
				struct mpam_vmsc, comp_list);

	class->props = vmsc->props;
}

static void mpam_enable_merge_vmsc_features(struct mpam_component *comp)
{
	struct mpam_vmsc *vmsc;
	struct mpam_msc_ris *ris;
	struct mpam_class *class = comp->class;

	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
		list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
			__vmsc_props_mismatch(vmsc, ris);
			class->nrdy_usec = max(class->nrdy_usec,
					       vmsc->msc->nrdy_usec);
		}
	}
}

static void mpam_enable_merge_class_features(struct mpam_component *comp)
{
	struct mpam_vmsc *vmsc;
	struct mpam_class *class = comp->class;

	list_for_each_entry(vmsc, &comp->vmsc, comp_list)
		__class_props_mismatch(class, vmsc);

	if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, class))
		mpam_clear_feature(mpam_feat_mbw_min, &class->props);
}

/*
 * Merge all the common resource features into class.
 * vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features()
 * as the first step so that mpam_enable_init_class_features() can initialise
 * the class with a representative set of features.
 * Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc
 * features to form the class features.
 * Other features are the min/max as appropriate.
 *
 * To avoid walking the whole tree twice, the class->nrdy_usec property is
 * updated when working with the vmsc as it is a max(), and doesn't need
 * initialising first.
 */
static void mpam_enable_merge_features(struct list_head *all_classes_list)
{
	struct mpam_class *class;
	struct mpam_component *comp;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry(class, all_classes_list, classes_list) {
		list_for_each_entry(comp, &class->components, class_list)
			mpam_enable_merge_vmsc_features(comp);

		mpam_enable_init_class_features(class);

		list_for_each_entry(comp, &class->components, class_list)
			mpam_enable_merge_class_features(comp);
	}
}

static char *mpam_errcode_names[16] = {
	[MPAM_ERRCODE_NONE]			= "No error",
	[MPAM_ERRCODE_PARTID_SEL_RANGE]		= "PARTID_SEL_Range",
	[MPAM_ERRCODE_REQ_PARTID_RANGE]		= "Req_PARTID_Range",
	[MPAM_ERRCODE_MSMONCFG_ID_RANGE]	= "MSMONCFG_ID_RANGE",
	[MPAM_ERRCODE_REQ_PMG_RANGE]		= "Req_PMG_Range",
	[MPAM_ERRCODE_MONITOR_RANGE]		= "Monitor_Range",
	[MPAM_ERRCODE_INTPARTID_RANGE]		= "intPARTID_Range",
	[MPAM_ERRCODE_UNEXPECTED_INTERNAL]	= "Unexpected_INTERNAL",
	[MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL]	= "Undefined_RIS_PART_SEL",
	[MPAM_ERRCODE_RIS_NO_CONTROL]		= "RIS_No_Control",
	[MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL]	= "Undefined_RIS_MON_SEL",
	[MPAM_ERRCODE_RIS_NO_MONITOR]		= "RIS_No_Monitor",
	[12 ... 15] = "Reserved"
};

/*
 * This will run as the threaded IRQ handler part when using MPAM-Fb, but
 * as the sole hard-IRQ handler for MMIO based accesses.
 */
static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc)
{
	u64 reg;
	int ret;
	u16 partid;
	u8 errcode, pmg, ris;

	if (WARN_ON_ONCE(!msc))
		return IRQ_NONE;

	if (msc->iface == MPAM_IFACE_MMIO &&
	    WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
					   &msc->accessibility)))
		return IRQ_NONE;

	ret = mpam_msc_read_esr(msc, &reg);
	if (ret) {
		pr_err_ratelimited("unknown error irq from msc:%u\n", msc->id);

		/* Try out best here ... */
		goto out_disable;
	}

	errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg);
	if (!errcode)
		return IRQ_NONE;

	/*
	 * Clear the level triggered IRQ. If that fails, we cannot do anything
	 * about it, so ignore any errors. We will disable the IRQ either on
	 * the device side or on the irqchip level next anyway.
	 */
	mpam_msc_clear_esr(msc);

	partid = FIELD_GET(MPAMF_ESR_PARTID_MON, reg);
	pmg = FIELD_GET(MPAMF_ESR_PMG, reg);
	ris = FIELD_GET(MPAMF_ESR_RIS, reg);

	pr_err_ratelimited("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n",
			   msc->id, mpam_errcode_names[errcode], partid, pmg,
			   ris);

out_disable:
	/*
	 * Disable this interrupt on the device side. If that fails, disable
	 * the IRQ on the irqchip level, as we must prevent further handler
	 * invocations. We only take the interrupt once anyway, as we
	 * are going to free the IRQ next, in mpam_disable().
	 */
	ret = mpam_disable_msc_ecr(msc);
	if (ret)
		disable_irq_nosync(irq);

	/* Check whether we are racing with the thread disabling MPAM. */
	if (!mpam_is_enabled())
		return IRQ_HANDLED;

	/*
	 * Schedule the teardown work. We have to defer it as we can't
	 * unregister the interrupt from the threaded part of a handler.
	 */
	mpam_disable_reason = "hardware error interrupt";
	schedule_work(&mpam_broken_work);

	return IRQ_HANDLED;
}

static irqreturn_t mpam_ppi_handler(int irq, void *dev_id)
{
	struct mpam_msc *msc = *(struct mpam_msc **)dev_id;

	return __mpam_irq_handler(irq, msc);
}

/*
 * MMIO based MSC accesses must run in non-preemptible context, as they
 * might have affinity requirements to check.
 * MPAM-Fb based MSC accesses must NOT run in hard-IRQ context, as they
 * can sleep.
 * So split the IRQ handling up, depending on the MSC access type.
 */
static irqreturn_t mpam_shared_hard_irq(int irq, void *dev_id)
{
	struct mpam_msc *msc = dev_id;

	if (msc->iface == MPAM_IFACE_MMIO)
		return __mpam_irq_handler(irq, msc);

	return IRQ_WAKE_THREAD;
}

static irqreturn_t mpam_shared_thread_irq(int irq, void *dev_id)
{
	struct mpam_msc *msc = dev_id;

	if (msc->iface == MPAM_IFACE_MMIO)
		return IRQ_HANDLED;

	return __mpam_irq_handler(irq, msc);
}

static int mpam_register_irqs(void)
{
	int err, irq;
	struct mpam_msc *msc;

	lockdep_assert_cpus_held();

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		irq = platform_get_irq_byname_optional(msc->pdev, "error");
		if (irq <= 0)
			continue;

		/* The MPAM spec says the interrupt can be SPI, PPI or LPI */
		/* We anticipate sharing the interrupt with other MSCs */
		if (irq_is_percpu(irq)) {
			if (msc->iface != MPAM_IFACE_MMIO) {
				dev_err(&msc->pdev->dev,
					"Only MMIO MSCs can use per-CPU interrupts\n");
				return -EINVAL;
			}
			err = request_percpu_irq(irq, &mpam_ppi_handler,
						 "mpam:msc:error",
						 msc->error_dev_id);
			if (err)
				return err;

			msc->reenable_error_ppi = irq;
			smp_call_function_many(&msc->accessibility,
					       &_enable_percpu_irq, &irq,
					       true);
		} else {
			err = devm_request_threaded_irq(&msc->pdev->dev, irq,
							&mpam_shared_hard_irq,
							&mpam_shared_thread_irq,
							IRQF_SHARED | IRQF_ONESHOT,
							"mpam:msc:error", msc);
			if (err)
				return err;
		}

		guard(mutex)(&msc->error_irq_lock);
		msc->error_irq_req = true;
		err = mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
		if (err)
			return err;
		msc->error_irq_hw_enabled = true;
	}

	return 0;
}

static void mpam_unregister_irqs(void)
{
	int irq;
	struct mpam_msc *msc;

	guard(cpus_read_lock)();
	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		irq = platform_get_irq_byname_optional(msc->pdev, "error");
		if (irq <= 0)
			continue;

		mutex_lock(&msc->error_irq_lock);
		if (msc->error_irq_hw_enabled) {
			if (!mpam_touch_msc(msc, mpam_disable_msc_ecr, msc))
				msc->error_irq_hw_enabled = false;
		}

		if (msc->error_irq_req) {
			if (irq_is_percpu(irq)) {
				msc->reenable_error_ppi = 0;
				free_percpu_irq(irq, msc->error_dev_id);
			} else {
				devm_free_irq(&msc->pdev->dev, irq, msc);
			}
			msc->error_irq_req = false;
		}
		mutex_unlock(&msc->error_irq_lock);
	}
}

static void __destroy_component_cfg(struct mpam_component *comp)
{
	struct mpam_msc *msc;
	struct mpam_vmsc *vmsc;
	struct mpam_msc_ris *ris;

	lockdep_assert_held(&mpam_list_lock);

	if (!comp->cfg)
		return;

	add_to_garbage(comp->cfg);
	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
		msc = vmsc->msc;

		if (mpam_mon_sel_lock(msc)) {
			list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
				if (ris->mbwu_state)
					add_to_garbage(ris->mbwu_state);
			}
			mpam_mon_sel_unlock(msc);
		}
	}
}

static void mpam_reset_component_cfg(struct mpam_component *comp)
{
	int i;
	struct mpam_props *cprops = &comp->class->props;

	mpam_assert_partid_sizes_fixed();

	if (!comp->cfg)
		return;

	for (i = 0; i <= mpam_partid_max; i++) {
		comp->cfg[i] = (struct mpam_config) {};
		if (cprops->cpbm_wd)
			comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0);
		if (cprops->mbw_pbm_bits)
			comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0);
		if (cprops->bwa_wd)
			comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd);
	}
}

static int __allocate_component_cfg(struct mpam_component *comp)
{
	struct mpam_vmsc *vmsc;

	mpam_assert_partid_sizes_fixed();

	if (comp->cfg)
		return 0;

	comp->cfg = kvzalloc_objs(*comp->cfg, mpam_partid_max + 1);
	if (!comp->cfg)
		return -ENOMEM;

	/*
	 * The array is free()d in one go, so only cfg[0]'s structure needs
	 * to be initialised.
	 */
	init_garbage(&comp->cfg[0].garbage);

	mpam_reset_component_cfg(comp);

	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
		struct mpam_msc *msc;
		struct mpam_msc_ris *ris;
		struct msmon_mbwu_state *mbwu_state;

		if (!vmsc->props.num_mbwu_mon)
			continue;

		msc = vmsc->msc;
		list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
			if (!ris->props.num_mbwu_mon)
				continue;

			mbwu_state = kzalloc_objs(*ris->mbwu_state,
						  ris->props.num_mbwu_mon);
			if (!mbwu_state) {
				__destroy_component_cfg(comp);
				return -ENOMEM;
			}

			init_garbage(&mbwu_state[0].garbage);

			if (mpam_mon_sel_lock(msc)) {
				ris->mbwu_state = mbwu_state;
				mpam_mon_sel_unlock(msc);
			}
		}
	}

	return 0;
}

static int mpam_allocate_config(void)
{
	struct mpam_class *class;
	struct mpam_component *comp;

	lockdep_assert_held(&mpam_list_lock);

	list_for_each_entry(class, &mpam_classes, classes_list) {
		list_for_each_entry(comp, &class->components, class_list) {
			int err = __allocate_component_cfg(comp);
			if (err)
				return err;
		}
	}

	return 0;
}

static void mpam_enable_once(void)
{
	int err;

	/*
	 * Once the cpuhp callbacks have been changed, mpam_partid_max can no
	 * longer change.
	 */
	spin_lock(&partid_max_lock);
	partid_max_published = true;
	spin_unlock(&partid_max_lock);

	/*
	 * If all the MSC have been probed, enabling the IRQs happens next.
	 * That involves cross-calling to a CPU that can reach the MSC, and
	 * the locks must be taken in this order:
	 */
	cpus_read_lock();
	mutex_lock(&mpam_list_lock);
	do {
		mpam_enable_merge_features(&mpam_classes);

		err = mpam_register_irqs();
		if (err) {
			pr_warn("Failed to register irqs: %d\n", err);
			break;
		}

		err = mpam_allocate_config();
		if (err) {
			pr_err("Failed to allocate configuration arrays.\n");
			break;
		}
	} while (0);
	mutex_unlock(&mpam_list_lock);
	cpus_read_unlock();

	if (!err) {
		err = mpam_resctrl_setup();
		if (err)
			pr_err("Failed to initialise resctrl: %d\n", err);
	}

	if (err) {
		mpam_disable_reason = "Failed to enable.";
		schedule_work(&mpam_broken_work);
		return;
	}

	static_branch_enable(&mpam_enabled);
	mpam_resctrl_enabled = true;
	mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline,
				      "mpam:online");

	/* Use printk() to avoid the pr_fmt adding the function name. */
	printk(KERN_INFO "MPAM enabled with %u PARTIDs and %u PMGs\n",
	       mpam_partid_max + 1, mpam_pmg_max + 1);
}

static int mpam_reset_component_locked(struct mpam_component *comp)
{
	struct mpam_vmsc *vmsc;

	lockdep_assert_cpus_held();
	mpam_assert_partid_sizes_fixed();

	mpam_reset_component_cfg(comp);

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		struct mpam_msc *msc = vmsc->msc;
		struct mpam_msc_ris *ris;
		int ret;

		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
					 srcu_read_lock_held(&mpam_srcu)) {
			if (!ris->in_reset_state) {
				ret = mpam_touch_msc(msc, mpam_reset_ris, ris);
				if (ret)
					return ret;
			}
			ris->in_reset_state = true;
		}
	}

	return 0;
}

int mpam_reset_class_locked(struct mpam_class *class)
{
	struct mpam_component *comp;
	int ret;

	lockdep_assert_cpus_held();

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(comp, &class->components, class_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		ret = mpam_reset_component_locked(comp);
		if (ret)
			return ret;
	}

	return 0;
}

static void mpam_reset_class(struct mpam_class *class)
{
	cpus_read_lock();
	mpam_reset_class_locked(class);
	cpus_read_unlock();
}

/*
 * Called in response to an error IRQ.
 * All of MPAMs errors indicate a software bug, restore any modified
 * controls to their reset values.
 */
void mpam_disable(struct work_struct *ignored)
{
	int idx;
	bool do_resctrl_exit;
	struct mpam_class *class;
	struct mpam_msc *msc, *tmp;

	if (mpam_is_enabled())
		static_branch_disable(&mpam_enabled);

	mutex_lock(&mpam_cpuhp_state_lock);
	if (mpam_cpuhp_state) {
		cpuhp_remove_state(mpam_cpuhp_state);
		mpam_cpuhp_state = 0;
	}

	/*
	 * Removing the cpuhp state called mpam_cpu_offline() and told resctrl
	 * all the CPUs are offline.
	 */
	do_resctrl_exit = mpam_resctrl_enabled;
	mpam_resctrl_enabled = false;
	mutex_unlock(&mpam_cpuhp_state_lock);

	if (do_resctrl_exit)
		mpam_resctrl_exit();

	mpam_unregister_irqs();

	idx = srcu_read_lock(&mpam_srcu);
	list_for_each_entry_srcu(class, &mpam_classes, classes_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		mpam_reset_class(class);
		if (do_resctrl_exit)
			mpam_resctrl_teardown_class(class);
	}
	srcu_read_unlock(&mpam_srcu, idx);

	mutex_lock(&mpam_list_lock);
	list_for_each_entry_safe(msc, tmp, &mpam_all_msc, all_msc_list)
		mpam_msc_destroy(msc);
	mutex_unlock(&mpam_list_lock);
	mpam_free_garbage();

	if (mpam_disable_errno || mpam_disable_mpam_fb_err)
		pr_err_once("MPAM disabled due to %s: %s, MPAM-Fb error %d\n",
			    mpam_disable_reason, errname(mpam_disable_errno),
			    mpam_disable_mpam_fb_err);
	else
		pr_err_once("MPAM disabled due to %s\n", mpam_disable_reason);
}

/*
 * Enable mpam once all devices have been probed.
 * Scheduled by mpam_discovery_cpu_online() once all devices have been created.
 * Also scheduled when new devices are probed when new CPUs come online.
 */
void mpam_enable(struct work_struct *work)
{
	static atomic_t once;
	struct mpam_msc *msc;
	bool all_devices_probed = true;

	/* Have we probed all the hw devices? */
	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		mutex_lock(&msc->probe_lock);
		if (!msc->probed)
			all_devices_probed = false;
		mutex_unlock(&msc->probe_lock);

		if (!all_devices_probed)
			break;
	}

	if (all_devices_probed && !atomic_fetch_inc(&once))
		mpam_enable_once();
}

#define maybe_update_config(cfg, feature, newcfg, member, changes) do { \
	if (mpam_has_feature(feature, newcfg) &&			\
	    (newcfg)->member != (cfg)->member) {			\
		(cfg)->member = (newcfg)->member;			\
		mpam_set_feature(feature, cfg);				\
									\
		(changes) = true;					\
	}								\
} while (0)

static bool mpam_update_config(struct mpam_config *cfg,
			       const struct mpam_config *newcfg)
{
	bool has_changes = false;

	maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes);
	maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes);
	maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes);

	return has_changes;
}

int mpam_apply_config(struct mpam_component *comp, u16 partid,
		      struct mpam_config *cfg)
{
	struct mpam_write_config_arg arg;
	struct mpam_msc_ris *ris;
	struct mpam_vmsc *vmsc;
	struct mpam_msc *msc;
	int ret;

	lockdep_assert_cpus_held();

	/* Don't pass in the current config! */
	WARN_ON_ONCE(&comp->cfg[partid] == cfg);

	if (!mpam_update_config(&comp->cfg[partid], cfg))
		return 0;

	arg.comp = comp;
	arg.partid = partid;

	guard(srcu)(&mpam_srcu);
	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
				 srcu_read_lock_held(&mpam_srcu)) {
		msc = vmsc->msc;

		guard(mutex)(&msc->cfg_lock);
		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
					 srcu_read_lock_held(&mpam_srcu)) {
			arg.ris = ris;
			ret = mpam_touch_msc(msc, __write_config, &arg);
			if (ret)
				return ret;
			ris->in_reset_state = false;
		}
	}

	return 0;
}

static int __init mpam_msc_driver_init(void)
{
	if (!system_supports_mpam())
		return -EOPNOTSUPP;

	init_srcu_struct(&mpam_srcu);

	fw_num_msc = acpi_mpam_count_msc();
	if (fw_num_msc <= 0) {
		pr_err("No MSC devices found in firmware\n");
		return -EINVAL;
	}

	return platform_driver_register(&mpam_msc_driver);
}

/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */
subsys_initcall(mpam_msc_driver_init);

#ifdef CONFIG_MPAM_KUNIT_TEST
#include "test_mpam_devices.c"
#endif