[PATCH 29/37] iommu/arm-smmu-v3: Add stall support for platform devices
Xu Zaibo
xuzaibo at huawei.com
Mon Feb 12 17:46:01 PST 2018
Hi,
On 2018/2/13 2:33, Jean-Philippe Brucker wrote:
> The SMMU provides a Stall model for handling page faults in platform
> devices. It is similar to PCI PRI, but doesn't require devices to have
> their own translation cache. Instead, faulting transactions are parked and
> the OS is given a chance to fix the page tables and retry the transaction.
>
> Enable stall for devices that support it (opt-in by firmware). When an
> event corresponds to a translation error, call the IOMMU fault handler. If
> the fault is recoverable, it will call us back to terminate or continue
> the stall.
>
> Note that this patch tweaks the iommu_fault_event and page_response_msg to
> extend the fault id field. Stall uses 16 bits of IDs whereas PCI PRI only
> uses 9.
For PCIe devices without ATC, can they use this Stall model?
Thanks.
Xu Zaibo
> Signed-off-by: Jean-Philippe Brucker <jean-philippe.brucker at arm.com>
> ---
> drivers/iommu/arm-smmu-v3.c | 175 +++++++++++++++++++++++++++++++++++++++++++-
> include/linux/iommu.h | 4 +-
> 2 files changed, 173 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
> index 2430b2140f8d..8b9f5dd06be0 100644
> --- a/drivers/iommu/arm-smmu-v3.c
> +++ b/drivers/iommu/arm-smmu-v3.c
> @@ -338,6 +338,15 @@
> #define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
> #define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
>
> +#define CMDQ_RESUME_0_SID_SHIFT 32
> +#define CMDQ_RESUME_0_SID_MASK 0xffffffffUL
> +#define CMDQ_RESUME_0_ACTION_SHIFT 12
> +#define CMDQ_RESUME_0_ACTION_TERM (0UL << CMDQ_RESUME_0_ACTION_SHIFT)
> +#define CMDQ_RESUME_0_ACTION_RETRY (1UL << CMDQ_RESUME_0_ACTION_SHIFT)
> +#define CMDQ_RESUME_0_ACTION_ABORT (2UL << CMDQ_RESUME_0_ACTION_SHIFT)
> +#define CMDQ_RESUME_1_STAG_SHIFT 0
> +#define CMDQ_RESUME_1_STAG_MASK 0xffffUL
> +
> #define CMDQ_SYNC_0_CS_SHIFT 12
> #define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
> #define CMDQ_SYNC_0_CS_IRQ (1UL << CMDQ_SYNC_0_CS_SHIFT)
> @@ -358,6 +367,31 @@
> #define EVTQ_0_ID_SHIFT 0
> #define EVTQ_0_ID_MASK 0xffUL
>
> +#define EVT_ID_TRANSLATION_FAULT 0x10
> +#define EVT_ID_ADDR_SIZE_FAULT 0x11
> +#define EVT_ID_ACCESS_FAULT 0x12
> +#define EVT_ID_PERMISSION_FAULT 0x13
> +
> +#define EVTQ_0_SSV (1UL << 11)
> +#define EVTQ_0_SSID_SHIFT 12
> +#define EVTQ_0_SSID_MASK 0xfffffUL
> +#define EVTQ_0_SID_SHIFT 32
> +#define EVTQ_0_SID_MASK 0xffffffffUL
> +#define EVTQ_1_STAG_SHIFT 0
> +#define EVTQ_1_STAG_MASK 0xffffUL
> +#define EVTQ_1_STALL (1UL << 31)
> +#define EVTQ_1_PRIV (1UL << 33)
> +#define EVTQ_1_EXEC (1UL << 34)
> +#define EVTQ_1_READ (1UL << 35)
> +#define EVTQ_1_S2 (1UL << 39)
> +#define EVTQ_1_CLASS_SHIFT 40
> +#define EVTQ_1_CLASS_MASK 0x3UL
> +#define EVTQ_1_TT_READ (1UL << 44)
> +#define EVTQ_2_ADDR_SHIFT 0
> +#define EVTQ_2_ADDR_MASK 0xffffffffffffffffUL
> +#define EVTQ_3_IPA_SHIFT 12
> +#define EVTQ_3_IPA_MASK 0xffffffffffUL
> +
> /* PRI queue */
> #define PRIQ_ENT_DWORDS 2
> #define PRIQ_MAX_SZ_SHIFT 8
> @@ -472,6 +506,13 @@ struct arm_smmu_cmdq_ent {
> enum pri_resp resp;
> } pri;
>
> + #define CMDQ_OP_RESUME 0x44
> + struct {
> + u32 sid;
> + u16 stag;
> + enum page_response_code resp;
> + } resume;
> +
> #define CMDQ_OP_CMD_SYNC 0x46
> struct {
> u32 msidata;
> @@ -545,6 +586,8 @@ struct arm_smmu_strtab_ent {
> bool assigned;
> struct arm_smmu_s1_cfg *s1_cfg;
> struct arm_smmu_s2_cfg *s2_cfg;
> +
> + bool can_stall;
> };
>
> struct arm_smmu_strtab_cfg {
> @@ -904,6 +947,21 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
> return -EINVAL;
> }
> break;
> + case CMDQ_OP_RESUME:
> + cmd[0] |= (u64)ent->resume.sid << CMDQ_RESUME_0_SID_SHIFT;
> + cmd[1] |= ent->resume.stag << CMDQ_RESUME_1_STAG_SHIFT;
> + switch (ent->resume.resp) {
> + case IOMMU_PAGE_RESP_INVALID:
> + case IOMMU_PAGE_RESP_FAILURE:
> + cmd[0] |= CMDQ_RESUME_0_ACTION_ABORT;
> + break;
> + case IOMMU_PAGE_RESP_SUCCESS:
> + cmd[0] |= CMDQ_RESUME_0_ACTION_RETRY;
> + break;
> + default:
> + return -EINVAL;
> + }
> + break;
> case CMDQ_OP_CMD_SYNC:
> if (ent->sync.msiaddr)
> cmd[0] |= CMDQ_SYNC_0_CS_IRQ;
> @@ -1065,6 +1123,35 @@ static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
> dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
> }
>
> +static int arm_smmu_page_response(struct iommu_domain *domain,
> + struct device *dev,
> + struct page_response_msg *resp)
> +{
> + int sid = dev->iommu_fwspec->ids[0];
> + struct arm_smmu_cmdq_ent cmd = {0};
> + struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
> +
> + if (master->ste.can_stall) {
> + cmd.opcode = CMDQ_OP_RESUME;
> + cmd.resume.sid = sid;
> + cmd.resume.stag = resp->page_req_group_id;
> + cmd.resume.resp = resp->resp_code;
> + } else {
> + /* TODO: put PRI response here */
> + return -EINVAL;
> + }
> +
> + arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
> + /*
> + * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
> + * RESUME consumption guarantees that the stalled transaction will be
> + * terminated... at some point in the future. PRI_RESP is fire and
> + * forget.
> + */
> +
> + return 0;
> +}
> +
> /* Stream table manipulation functions */
> static void
> arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
> @@ -1182,7 +1269,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
> STRTAB_STE_1_STRW_SHIFT);
>
> if (smmu->features & ARM_SMMU_FEAT_STALLS &&
> - !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
> + !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE) &&
> + !ste->can_stall)
> dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
>
> val |= (cfg->base & STRTAB_STE_0_S1CTXPTR_MASK
> @@ -1285,10 +1373,73 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
> return master;
> }
>
> +static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
> +{
> + struct arm_smmu_master_data *master;
> + u8 type = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
> + u32 sid = evt[0] >> EVTQ_0_SID_SHIFT & EVTQ_0_SID_MASK;
> +
> + struct iommu_fault_event fault = {
> + .page_req_group_id = evt[1] >> EVTQ_1_STAG_SHIFT & EVTQ_1_STAG_MASK,
> + .addr = evt[2] >> EVTQ_2_ADDR_SHIFT & EVTQ_2_ADDR_MASK,
> + .last_req = true,
> + };
> +
> + switch (type) {
> + case EVT_ID_TRANSLATION_FAULT:
> + case EVT_ID_ADDR_SIZE_FAULT:
> + case EVT_ID_ACCESS_FAULT:
> + fault.reason = IOMMU_FAULT_REASON_PTE_FETCH;
> + break;
> + case EVT_ID_PERMISSION_FAULT:
> + fault.reason = IOMMU_FAULT_REASON_PERMISSION;
> + break;
> + default:
> + /* TODO: report other unrecoverable faults. */
> + return -EFAULT;
> + }
> +
> + /* Stage-2 is always pinned at the moment */
> + if (evt[1] & EVTQ_1_S2)
> + return -EFAULT;
> +
> + master = arm_smmu_find_master(smmu, sid);
> + if (!master)
> + return -EINVAL;
> +
> + /*
> + * The domain is valid until the fault returns, because detach() flushes
> + * the fault queue.
> + */
> + if (evt[1] & EVTQ_1_STALL)
> + fault.type = IOMMU_FAULT_PAGE_REQ;
> + else
> + fault.type = IOMMU_FAULT_DMA_UNRECOV;
> +
> + if (evt[1] & EVTQ_1_READ)
> + fault.prot |= IOMMU_FAULT_READ;
> + else
> + fault.prot |= IOMMU_FAULT_WRITE;
> +
> + if (evt[1] & EVTQ_1_EXEC)
> + fault.prot |= IOMMU_FAULT_EXEC;
> +
> + if (evt[1] & EVTQ_1_PRIV)
> + fault.prot |= IOMMU_FAULT_PRIV;
> +
> + if (evt[0] & EVTQ_0_SSV) {
> + fault.pasid_valid = true;
> + fault.pasid = evt[0] >> EVTQ_0_SSID_SHIFT & EVTQ_0_SSID_MASK;
> + }
> +
> + /* Report to device driver or populate the page tables */
> + return iommu_report_device_fault(master->dev, &fault);
> +}
> +
> /* IRQ and event handlers */
> static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
> {
> - int i;
> + int i, ret;
> int num_handled = 0;
> struct arm_smmu_device *smmu = dev;
> struct arm_smmu_queue *q = &smmu->evtq.q;
> @@ -1300,12 +1451,19 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
> while (!queue_remove_raw(q, evt)) {
> u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
>
> + spin_unlock(&q->wq.lock);
> + ret = arm_smmu_handle_evt(smmu, evt);
> + spin_lock(&q->wq.lock);
> +
> if (++num_handled == queue_size) {
> q->batch++;
> wake_up_locked(&q->wq);
> num_handled = 0;
> }
>
> + if (!ret)
> + continue;
> +
> dev_info(smmu->dev, "event 0x%02x received:\n", id);
> for (i = 0; i < ARRAY_SIZE(evt); ++i)
> dev_info(smmu->dev, "\t0x%016llx\n",
> @@ -1442,7 +1600,9 @@ static int arm_smmu_flush_queues(struct notifier_block *nb,
> master = dev->iommu_fwspec->iommu_priv;
>
> if (master) {
> - /* TODO: add support for PRI and Stall */
> + if (master->ste.can_stall)
> + arm_smmu_flush_queue(smmu, &smmu->evtq.q, "evtq");
> + /* TODO: add support for PRI */
> return 0;
> }
>
> @@ -1756,7 +1916,8 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
> .order = master->ssid_bits,
> .sync = &arm_smmu_ctx_sync,
> .arm_smmu = {
> - .stall = !!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE),
> + .stall = !!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE) ||
> + master->ste.can_stall,
> .asid_bits = smmu->asid_bits,
> .hw_access = !!(smmu->features & ARM_SMMU_FEAT_HA),
> .hw_dirty = !!(smmu->features & ARM_SMMU_FEAT_HD),
> @@ -2296,6 +2457,11 @@ static int arm_smmu_add_device(struct device *dev)
>
> master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
>
> + if (fwspec->can_stall && smmu->features & ARM_SMMU_FEAT_STALLS) {
> + master->can_fault = true;
> + master->ste.can_stall = true;
> + }
> +
> group = iommu_group_get_for_dev(dev);
> if (!IS_ERR(group)) {
> arm_smmu_insert_master(smmu, master);
> @@ -2435,6 +2601,7 @@ static struct iommu_ops arm_smmu_ops = {
> .mm_attach = arm_smmu_mm_attach,
> .mm_detach = arm_smmu_mm_detach,
> .mm_invalidate = arm_smmu_mm_invalidate,
> + .page_response = arm_smmu_page_response,
> .map = arm_smmu_map,
> .unmap = arm_smmu_unmap,
> .map_sg = default_iommu_map_sg,
> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
> index 37c3b9d087ce..f5c2f4be2b42 100644
> --- a/include/linux/iommu.h
> +++ b/include/linux/iommu.h
> @@ -227,7 +227,7 @@ struct page_response_msg {
> u32 pasid;
> enum page_response_code resp_code;
> u32 pasid_present:1;
> - u32 page_req_group_id : 9;
> + u32 page_req_group_id;
> enum page_response_type type;
> u32 private_data;
> };
> @@ -421,7 +421,7 @@ struct iommu_fault_event {
> enum iommu_fault_reason reason;
> u64 addr;
> u32 pasid;
> - u32 page_req_group_id : 9;
> + u32 page_req_group_id;
> u32 last_req : 1;
> u32 pasid_valid : 1;
> u32 prot;
More information about the linux-arm-kernel
mailing list