[PATCH 7/9] iommu/arm-smmu-v3: Directly encode CMDQ_OP_ATC_INV
Jason Gunthorpe
jgg at nvidia.com
Fri May 1 07:29:16 PDT 2026
Add a new command make function and convert all the places using
ATC_INV.
Split out full invalidation to directly make the cmd instead of
overloading size=0 to mean full invalidation.
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 59 ++++++++-------------
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 27 +++++++---
2 files changed, 40 insertions(+), 46 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f9c25ca9a9e7b8..0cdf0752ff6d62 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -308,14 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
case CMDQ_OP_TLBI_EL2_ASID:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
break;
- case CMDQ_OP_ATC_INV:
- cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
- cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
- cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
- cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
- cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
- cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
- break;
case CMDQ_OP_CMD_SYNC:
if (ent->sync.msiaddr) {
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
@@ -2371,9 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
return IRQ_WAKE_THREAD;
}
-static void
-arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
- struct arm_smmu_cmdq_ent *cmd)
+static struct arm_smmu_cmd
+arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
{
size_t log2_span;
size_t span_mask;
@@ -2395,17 +2386,6 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
* This has the unpleasant side-effect of invalidating all PASID-tagged
* ATC entries within the address range.
*/
- *cmd = (struct arm_smmu_cmdq_ent) {
- .opcode = CMDQ_OP_ATC_INV,
- .substream_valid = (ssid != IOMMU_NO_PASID),
- .atc.ssid = ssid,
- };
-
- if (!size) {
- cmd->atc.size = ATC_INV_SIZE_ALL;
- return;
- }
-
page_start = iova >> inval_grain_shift;
page_end = (iova + size - 1) >> inval_grain_shift;
@@ -2434,24 +2414,25 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
page_start &= ~span_mask;
- cmd->atc.addr = page_start << inval_grain_shift;
- cmd->atc.size = log2_span;
+ return arm_smmu_make_cmd_atc_inv(sid, ssid,
+ page_start << inval_grain_shift,
+ log2_span);
}
static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
ioasid_t ssid)
{
int i;
- struct arm_smmu_cmdq_ent cmd;
+ struct arm_smmu_cmd cmd;
struct arm_smmu_cmdq_batch cmds;
- arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
-
- arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
- for (i = 0; i < master->num_streams; i++) {
- cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
- }
+ cmd = arm_smmu_make_cmd_atc_inv_all(0, IOMMU_NO_PASID);
+ arm_smmu_cmdq_batch_init_cmd(master->smmu, &cmds, &cmd);
+ for (i = 0; i < master->num_streams; i++)
+ arm_smmu_cmdq_batch_add_cmd(
+ master->smmu, &cmds,
+ arm_smmu_make_cmd_atc_inv_all(master->streams[i].id,
+ ssid));
return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
}
@@ -2650,14 +2631,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
break;
case INV_TYPE_ATS:
- arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd);
- cmd.atc.sid = cur->id;
- arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+ arm_smmu_cmdq_batch_add_cmd(
+ smmu, &cmds,
+ arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid,
+ iova, size));
break;
case INV_TYPE_ATS_FULL:
- arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
- cmd.atc.sid = cur->id;
- arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+ arm_smmu_cmdq_batch_add_cmd(
+ smmu, &cmds,
+ arm_smmu_make_cmd_atc_inv_all(cur->id,
+ IOMMU_NO_PASID));
break;
default:
WARN_ON_ONCE(1);
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 10b3d95d9ee660..194f73cabef5c9 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -552,6 +552,25 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid,
return cmd;
}
+static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv(u32 sid, u32 ssid,
+ u64 addr, u8 size)
+{
+ struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_ATC_INV);
+
+ cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssid != IOMMU_NO_PASID) |
+ FIELD_PREP(CMDQ_ATC_0_SSID, ssid) |
+ FIELD_PREP(CMDQ_ATC_0_SID, sid);
+ cmd.data[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, size) |
+ (addr & CMDQ_ATC_1_ADDR_MASK);
+ return cmd;
+}
+
+static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid,
+ u32 ssid)
+{
+ return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL);
+}
+
/* Event queue */
#define EVTQ_ENT_SZ_SHIFT 5
#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
@@ -630,14 +649,6 @@ struct arm_smmu_cmdq_ent {
u64 addr;
} tlbi;
- struct {
- u32 sid;
- u32 ssid;
- u64 addr;
- u8 size;
- bool global;
- } atc;
-
struct {
u64 msiaddr;
} sync;
--
2.43.0
More information about the linux-arm-kernel
mailing list