[PATCH v5 2/9] iommu/arm-smmu-v3: Pass the parameters for the invalidation in a struct
Jason Gunthorpe
jgg at nvidia.com
Tue Sep 1 10:49:51 PDT 2026
These parameters go to a lot of different functions and the next
patches will add more. Put them into a struct to keep things tidy.
Reviewed-by: Nicolin Chen <nicolinc at nvidia.com>
Reviewed-by: Mostafa Saleh <smostafa at google.com>
Tested-by: Nicolin Chen <nicolinc at nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 90 ++++++++++-----------
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 9 +++
2 files changed, 54 insertions(+), 45 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d6896e25b6632a..03723bb00de412 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2363,8 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
return IRQ_WAKE_THREAD;
}
-static struct arm_smmu_cmd
-arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
+static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid,
+ struct arm_smmu_tlbi *tlbi)
{
size_t log2_span;
size_t span_mask;
@@ -2386,8 +2386,8 @@ arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
* This has the unpleasant side-effect of invalidating all PASID-tagged
* ATC entries within the address range.
*/
- page_start = iova >> inval_grain_shift;
- page_end = (iova + size - 1) >> inval_grain_shift;
+ page_start = tlbi->iova >> inval_grain_shift;
+ page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift;
/*
* In an ATS Invalidate Request, the address must be aligned on the
@@ -2462,20 +2462,23 @@ static void arm_smmu_tlb_inv_context(void *cookie)
static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
- struct arm_smmu_cmd *cmd, bool leaf,
- unsigned long iova, size_t size,
- size_t granule, size_t pgsize)
+ struct arm_smmu_cmd *cmd,
+ struct arm_smmu_tlbi *tlbi,
+ size_t pgsize)
{
- unsigned long end = iova + size, num_pages = 0, tg = pgsize;
+ size_t inv_range = tlbi->iopte_size;
+ unsigned long iova = tlbi->iova;
+ unsigned long end = iova + tlbi->size;
+ unsigned long num_pages = 0;
+ unsigned int tg = pgsize;
u64 orig_data0 = cmd->data[0];
- size_t inv_range = granule;
u8 ttl = 0, tg_enc = 0;
- if (WARN_ON_ONCE(!size))
+ if (WARN_ON_ONCE(!tlbi->size))
return;
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
- num_pages = size >> tg;
+ num_pages = tlbi->size >> tg;
/* Convert page size of 12,14,16 (log2) to 1,2,3 */
tg_enc = (tg - 10) / 2;
@@ -2488,8 +2491,8 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
* want to use a range command, so avoid the SVA corner case
* where both scale and num could be 0 as well.
*/
- if (leaf)
- ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
+ if (tlbi->leaf_only)
+ ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tg - 3));
else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
num_pages++;
}
@@ -2528,7 +2531,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
* command and something would be very broken if iova had them
* set.
*/
- cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) |
+ cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) |
FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
(iova & ~GENMASK_U64(11, 0));
@@ -2568,13 +2571,13 @@ static bool arm_smmu_cmdq_batch_add_ril(struct arm_smmu_device *smmu,
return true;
}
-static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
- size_t granule)
+static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu,
+ struct arm_smmu_tlbi *tlbi)
{
size_t max_tlbi_ops;
/* 0 size means invalidate all */
- if (!size || size == SIZE_MAX)
+ if (!tlbi->size || tlbi->size == SIZE_MAX)
return true;
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV)
@@ -2587,32 +2590,31 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
* invalidation feature, where there can be too many per-granule TLBIs,
* resulting in a soft lockup.
*/
- max_tlbi_ops = 1 << (ilog2(granule) - 3);
- return size >= max_tlbi_ops * granule;
+ max_tlbi_ops = 1 << (ilog2(tlbi->iopte_size) - 3);
+ return tlbi->size >= max_tlbi_ops * tlbi->iopte_size;
}
/* Used by non INV_TYPE_ATS* invalidations */
static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
- bool single_ril, bool leaf,
- unsigned long iova, size_t size,
- unsigned int granule)
+ struct arm_smmu_tlbi *tlbi)
{
struct arm_smmu_cmd nsize_cmd;
- if (arm_smmu_inv_size_too_big(inv->smmu, size, granule))
+ if (arm_smmu_inv_size_too_big(inv->smmu, tlbi))
goto full_inv;
- if (single_ril && size > granule) {
- if (!arm_smmu_cmdq_batch_add_ril(inv->smmu, cmds, cmd, iova,
- size, inv->pgsize))
+ if (tlbi->has_cont && tlbi->size > tlbi->iopte_size &&
+ (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RIL)) {
+ if (!arm_smmu_cmdq_batch_add_ril(inv->smmu, cmds, cmd,
+ tlbi->iova, tlbi->size,
+ inv->pgsize))
goto full_inv;
return;
}
- arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, iova, size,
- granule, inv->pgsize);
+ arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi, inv->pgsize);
return;
full_inv:
@@ -2637,10 +2639,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
return false;
}
-static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
- unsigned long iova, size_t size,
- unsigned int granule, bool single_ril,
- bool leaf)
+static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi,
+ struct arm_smmu_invs *invs)
{
struct arm_smmu_cmdq_batch cmds = {};
struct arm_smmu_inv *cur;
@@ -2670,18 +2670,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
case INV_TYPE_S1_ASID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
- arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, single_ril,
- leaf, iova, size, granule);
+ arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi);
break;
case INV_TYPE_S2_VMID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
0, cur->id);
- arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, single_ril,
- leaf, iova, size, granule);
+ arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi);
break;
case INV_TYPE_S2_VMID_S1_CLEAR:
/* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
- if (arm_smmu_inv_size_too_big(cur->smmu, size, granule))
+ if (arm_smmu_inv_size_too_big(cur->smmu, tlbi))
break;
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
@@ -2692,7 +2690,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid,
- iova, size));
+ tlbi));
break;
case INV_TYPE_ATS_FULL:
arm_smmu_cmdq_batch_add_cmd(
@@ -2723,10 +2721,14 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long iova, size_t size,
unsigned int granule, bool leaf)
{
+ struct arm_smmu_tlbi tlbi = {
+ .iova = iova,
+ .size = size,
+ .iopte_size = granule,
+ .has_cont = smmu_domain->stage == ARM_SMMU_DOMAIN_SVA,
+ .leaf_only = leaf,
+ };
struct arm_smmu_invs *invs;
- bool single_ril =
- smmu_domain->stage == ARM_SMMU_DOMAIN_SVA &&
- (smmu_domain->smmu->options & ARM_SMMU_OPT_FULL_CONT_RIL);
/*
* An invalidation request must follow some IOPTE change and then load
@@ -2766,12 +2768,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long flags;
read_lock_irqsave(&invs->rwlock, flags);
- __arm_smmu_domain_inv_range(invs, iova, size, granule,
- single_ril, leaf);
+ __arm_smmu_domain_inv_range(&tlbi, invs);
read_unlock_irqrestore(&invs->rwlock, flags);
} else {
- __arm_smmu_domain_inv_range(invs, iova, size, granule,
- single_ril, leaf);
+ __arm_smmu_domain_inv_range(&tlbi, invs);
}
rcu_read_unlock();
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 065d76eb148119..d3e81056f24afc 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -805,6 +805,15 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
return new_invs;
}
+struct arm_smmu_tlbi {
+ unsigned long iova;
+ size_t size;
+ /* page or block size of the leaf iopte */
+ unsigned int iopte_size;
+ bool has_cont;
+ bool leaf_only;
+};
+
struct arm_smmu_evtq {
struct arm_smmu_queue q;
struct iopf_queue *iopf;
--
2.43.0
More information about the linux-arm-kernel
mailing list