[PATCH v10 11/15] iommu/tegra241-cmdqv: Add a helper to quiesce VCMDQs
Pranjal Shrivastava
praan at google.com
Tue Sep 8 10:17:07 PDT 2026
The tegra241-cmdqv driver supports vCMDQs which need to be quiesced using
the STOP_FLAG. The current driver implementation only uses VINTF0 for
vCMDQs owned by the kernel which need to be stopped. Add a helper that
sets the CMDQ_PROD_STOP_FLAG on these vCMDQs.
Consolidate this logic by renaming the implementation hook to
quiesce_and_drain_queues and ensuring that the tegra241-cmdqv driver
gates all active local virtual queues before starting the drain loop.
Additionally, clear the STOP_FLAG in tegra241_vcmdq_hw_init() as a part
of tegra241_cmdqv_hw_reset().
Suggested-by: Nicolin Chen <nicolinc at nvidia.com>
Signed-off-by: Pranjal Shrivastava <praan at google.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 +-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +-
.../iommu/arm/arm-smmu-v3/tegra241-cmdqv.c | 77 ++++++++++++++++++-
3 files changed, 76 insertions(+), 7 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 65939a9b0619..7eb939888735 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1077,8 +1077,8 @@ static int __maybe_unused arm_smmu_drain_cmdqs(struct arm_smmu_device *smmu)
ret = arm_smmu_drain_queue(smmu, &smmu->cmdq.q, true);
/* Drain all implementation-specific queues */
- if (smmu->impl_ops && smmu->impl_ops->drain_queues) {
- err = smmu->impl_ops->drain_queues(smmu);
+ if (smmu->impl_ops && smmu->impl_ops->quiesce_and_drain_queues) {
+ err = smmu->impl_ops->quiesce_and_drain_queues(smmu);
if (err)
ret = err;
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 794b258550dd..0a841441cc44 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -896,7 +896,7 @@ struct arm_smmu_impl_ops {
size_t (*get_viommu_size)(enum iommu_viommu_type viommu_type);
int (*vsmmu_init)(struct arm_vsmmu *vsmmu,
const struct iommu_user_data *user_data);
- int (*drain_queues)(struct arm_smmu_device *smmu);
+ int (*quiesce_and_drain_queues)(struct arm_smmu_device *smmu);
};
/* An SMMUv3 instance */
diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
index c7989fcd2c62..61847f2802a3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -447,6 +447,54 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
return &vcmdq->cmdq;
}
+static void tegra241_cmdqv_quiesce_vintf0_lvcmdqs(struct arm_smmu_device *smmu)
+{
+ struct tegra241_cmdqv *cmdqv =
+ container_of(smmu, struct tegra241_cmdqv, smmu);
+ struct tegra241_vintf *vintf = cmdqv->vintfs[0];
+ u16 lidx;
+
+ if (!READ_ONCE(vintf->enabled))
+ return;
+
+ for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) {
+ struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
+
+ if (!vcmdq || !READ_ONCE(vcmdq->enabled))
+ continue;
+
+ atomic_or(CMDQ_PROD_STOP_FLAG, &vcmdq->cmdq.q.llq.atomic.prod);
+ }
+}
+
+static void tegra241_vcmdq_wait_quiescent(struct arm_smmu_device *smmu,
+ struct tegra241_vcmdq *vcmdq)
+{
+ u32 target = READ_ONCE(vcmdq->cmdq.q.llq.prod) & CMDQ_PROD_IDX_MASK;
+ int timeout = ARM_SMMU_POLL_TIMEOUT_US;
+
+ /* Wait for the last committed owner to reach the hardware */
+ while (atomic_read(&vcmdq->cmdq.owner_prod) != target && timeout) {
+ udelay(1);
+ timeout--;
+ }
+
+ if (!timeout)
+ dev_err(smmu->dev, "vintf0 lvcmdq%u owner wait timeout\n",
+ vcmdq->lidx);
+
+ /* Wait for queue lock to be released */
+ timeout = ARM_SMMU_POLL_TIMEOUT_US;
+ while (atomic_read(&vcmdq->cmdq.lock) != 0 && timeout) {
+ udelay(1);
+ timeout--;
+ }
+
+ if (!timeout)
+ dev_err(smmu->dev, "vintf0 lvcmdq%u lock wait timeout\n",
+ vcmdq->lidx);
+}
+
static int tegra241_cmdqv_drain_vintf0_lvcmdqs(struct arm_smmu_device *smmu)
{
struct tegra241_cmdqv *cmdqv =
@@ -467,6 +515,21 @@ static int tegra241_cmdqv_drain_vintf0_lvcmdqs(struct arm_smmu_device *smmu)
if (!READ_ONCE(vintf->enabled))
return 0;
+ /*
+ * Gate all vCMDQs by setting the STOP_FLAG in a separate,
+ * initial loop to ensure no new commands can be submitted
+ * to any secondary queue while we are waiting to drain them.
+ *
+ * Client devices are suspended at this point due to devlinks,
+ * ensuring no concurrent command submissions race with this
+ * drain sequence.
+ */
+ tegra241_cmdqv_quiesce_vintf0_lvcmdqs(smmu);
+
+ /* Ensure all CPUs observe the STOP_FLAG before draining */
+ smp_mb();
+
+ /* Now that all queues are safely gated, drain them sequentially. */
for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) {
struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
int rc;
@@ -474,6 +537,9 @@ static int tegra241_cmdqv_drain_vintf0_lvcmdqs(struct arm_smmu_device *smmu)
if (!vcmdq || !READ_ONCE(vcmdq->enabled))
continue;
+ /* Wait for the last committed owner to reach the hardware */
+ tegra241_vcmdq_wait_quiescent(smmu, vcmdq);
+
rc = arm_smmu_drain_queue(smmu, &vcmdq->cmdq.q, true);
if (rc) {
/*
@@ -489,7 +555,7 @@ static int tegra241_cmdqv_drain_vintf0_lvcmdqs(struct arm_smmu_device *smmu)
}
/* Avoid consuming stale commands on resume */
- vcmdq->cmdq.q.llq.cons = vcmdq->cmdq.q.llq.prod;
+ vcmdq->cmdq.q.llq.cons = vcmdq->cmdq.q.llq.prod & CMDQ_PROD_IDX_MASK;
}
return ret;
@@ -566,7 +632,6 @@ static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq)
/* Configure and enable VCMDQ */
writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE));
-
/*
* HW Registers reset to 0 when power-cycled. Restore them from their
* SW copies to prevent executing stale/ghost commands after resume.
@@ -574,7 +639,8 @@ static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq)
* to the Guests since the relevant frameworks (IOMMUFD / VFIO) hold
* active PM references preventing suspend while VMs are active.
*/
- writel_relaxed(vcmdq->cmdq.q.llq.prod, REG_VCMDQ_PAGE0(vcmdq, PROD));
+ writel_relaxed(vcmdq->cmdq.q.llq.prod & CMDQ_PROD_IDX_MASK,
+ REG_VCMDQ_PAGE0(vcmdq, PROD));
writel_relaxed(vcmdq->cmdq.q.llq.cons, REG_VCMDQ_PAGE0(vcmdq, CONS));
ret = vcmdq_write_config(vcmdq, VCMDQ_EN);
@@ -587,6 +653,9 @@ static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq)
return ret;
}
+ /* Clear the CMDQ_PROD_STOP_FLAG */
+ atomic_andnot(CMDQ_PROD_STOP_FLAG, &vcmdq->cmdq.q.llq.atomic.prod);
+
dev_dbg(vcmdq->cmdqv->dev, "%sinited\n", h);
return 0;
}
@@ -963,7 +1032,7 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = {
.device_reset = tegra241_cmdqv_hw_reset,
.device_disable = tegra241_cmdqv_hw_disable,
.device_remove = tegra241_cmdqv_remove,
- .drain_queues = tegra241_cmdqv_drain_vintf0_lvcmdqs,
+ .quiesce_and_drain_queues = tegra241_cmdqv_drain_vintf0_lvcmdqs,
/* For user-space use */
.hw_info = tegra241_cmdqv_hw_info,
.get_viommu_size = tegra241_cmdqv_get_vintf_size,
--
2.55.0.979.g7e5102b832-goog
More information about the linux-arm-kernel
mailing list