[PATCH v4 11/27] iommu/arm-smmu-v3: Lift CD programming out of the SVA notifier code
Jason Gunthorpe
jgg at nvidia.com
Fri Jan 26 10:15:13 PST 2024
ops->set_dev_pasid()/ops->remove_dev_pasid() should work on a single CD
table entry, the one that was actually passed in to the function. The
current iterating over the master's list is a hold over from the prior
design where the CD table was part of the S1 domain.
Lift this code up and out so that we setup the CD only once for the
correct thing.
The SVA code "works" under a single configuration:
- The RID domain is a S1 domain
- The programmed PASID is the mm->pasid
- Nothing changes while SVA is running (sva_enable)
Invalidation will still iterate over the S1 domain's master list. That
remains OK after this change, we may do harmless extra ATS invalidations
for PASIDs that don't need it.
Signed-off-by: Jason Gunthorpe <jgg at nvidia.com>
---
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 58 ++++++-------------
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 29 ++++++++++
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 6 ++
3 files changed, 53 insertions(+), 40 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 7039620c4b2e55..973a5219c519cd 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -351,10 +351,8 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
struct mm_struct *mm)
{
int ret;
- unsigned long flags;
struct arm_smmu_ctx_desc *cd;
struct arm_smmu_mmu_notifier *smmu_mn;
- struct arm_smmu_master *master;
list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
if (smmu_mn->mn.mm == mm) {
@@ -384,35 +382,9 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
goto err_free_cd;
}
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head) {
- struct arm_smmu_cd target;
- struct arm_smmu_cd *cdptr;
-
- cdptr = arm_smmu_get_cd_ptr(master, mm_get_enqcmd_pasid(mm));
- if (!cdptr) {
- ret = -ENOMEM;
- list_for_each_entry_from_reverse(
- master, &smmu_domain->devices, domain_head)
- arm_smmu_clear_cd(master,
- mm_get_enqcmd_pasid(mm));
- break;
- }
-
- arm_smmu_make_sva_cd(&target, master, mm, cd->asid);
- arm_smmu_write_cd_entry(master, mm_get_enqcmd_pasid(mm), cdptr,
- &target);
- }
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
- if (ret)
- goto err_put_notifier;
-
list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers);
return smmu_mn;
-err_put_notifier:
- /* Frees smmu_mn */
- mmu_notifier_put(&smmu_mn->mn);
err_free_cd:
arm_smmu_free_shared_cd(cd);
return ERR_PTR(ret);
@@ -423,19 +395,12 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
struct mm_struct *mm = smmu_mn->mn.mm;
struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
- struct arm_smmu_master *master;
- unsigned long flags;
if (!refcount_dec_and_test(&smmu_mn->refs))
return;
list_del(&smmu_mn->list);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head)
- arm_smmu_clear_cd(master, mm_get_enqcmd_pasid(mm));
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
-
/*
* If we went through clear(), we've already invalidated, and no
* new TLB entry can have been formed.
@@ -451,7 +416,8 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
arm_smmu_free_shared_cd(cd);
}
-static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
+static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
+ struct arm_smmu_cd *target)
{
int ret;
struct arm_smmu_bond *bond;
@@ -481,6 +447,7 @@ static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
}
list_add(&bond->list, &master->bonds);
+ arm_smmu_make_sva_cd(target, master, mm, bond->smmu_mn->cd->asid);
return 0;
err_free_bond:
@@ -643,6 +610,8 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct arm_smmu_bond *bond = NULL, *t;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+ arm_smmu_remove_pasid(master, to_smmu_domain(domain), id);
+
mutex_lock(&sva_lock);
list_for_each_entry(t, &master->bonds, list) {
if (t->mm == mm) {
@@ -662,17 +631,26 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
int ret = 0;
struct mm_struct *mm = domain->mm;
+ struct arm_smmu_cd target;
if (mm_get_enqcmd_pasid(mm) != id)
return -EINVAL;
- mutex_lock(&sva_lock);
- ret = __arm_smmu_sva_bind(dev, mm);
- mutex_unlock(&sva_lock);
+ if (!arm_smmu_get_cd_ptr(master, id))
+ return -ENOMEM;
- return ret;
+ mutex_lock(&sva_lock);
+ ret = __arm_smmu_sva_bind(dev, mm, &target);
+ mutex_unlock(&sva_lock);
+ if (ret)
+ return ret;
+
+ /* This cannot fail since we preallocated the cdptr */
+ arm_smmu_set_pasid(master, to_smmu_domain(domain), id, &target);
+ return 0;
}
static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 8bd072ab0f258f..712795232fcfa9 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2656,6 +2656,35 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return 0;
}
+static bool arm_smmu_is_s1_domain(struct iommu_domain *domain)
+{
+ if (!domain || !(domain->type & __IOMMU_DOMAIN_PAGING))
+ return false;
+ return to_smmu_domain(domain)->stage == ARM_SMMU_DOMAIN_S1;
+}
+
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd)
+{
+ struct arm_smmu_cd *cdptr;
+
+ if (!arm_smmu_is_s1_domain(iommu_get_domain_for_dev(master->dev)))
+ return -ENODEV;
+
+ cdptr = arm_smmu_get_cd_ptr(master, pasid);
+ if (!cdptr)
+ return -ENOMEM;
+ arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
+ return 0;
+}
+
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid)
+{
+ arm_smmu_clear_cd(master, pasid);
+}
+
static int arm_smmu_attach_dev_ste(struct device *dev,
struct arm_smmu_ste *ste)
{
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 1f52f861281eb6..bc6fa08cc788a9 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -756,6 +756,12 @@ void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
struct arm_smmu_cd *cdptr,
const struct arm_smmu_cd *target);
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd);
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid);
+
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,
--
2.43.0
More information about the linux-arm-kernel
mailing list