[PATCH 2/5] iommu/mediatek: Add mt8173 IOMMU driver

yong.wu at mediatek.com yong.wu at mediatek.com
Fri Mar 6 02:48:17 PST 2015


From: Yong Wu <yong.wu at mediatek.com>

This patch adds support for mediatek m4u (MultiMedia Memory Management Unit).
Currently this only supports m4u gen 2 with 2 levels of page table on mt8173.

Signed-off-by: Yong Wu <yong.wu at mediatek.com>
---
 drivers/iommu/Kconfig               |  11 +
 drivers/iommu/Makefile              |   1 +
 drivers/iommu/mtk_iommu.c           | 754 ++++++++++++++++++++++++++++++++++++
 drivers/iommu/mtk_iommu.h           |  73 ++++
 drivers/iommu/mtk_iommu_pagetable.c | 439 +++++++++++++++++++++
 drivers/iommu/mtk_iommu_pagetable.h |  49 +++
 6 files changed, 1327 insertions(+)
 create mode 100644 drivers/iommu/mtk_iommu.c
 create mode 100644 drivers/iommu/mtk_iommu.h
 create mode 100644 drivers/iommu/mtk_iommu_pagetable.c
 create mode 100644 drivers/iommu/mtk_iommu_pagetable.h

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 19027bb..e63f5b6 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -326,4 +326,15 @@ config ARM_SMMU
 	  Say Y here if your SoC includes an IOMMU device implementing
 	  the ARM SMMU architecture.
 
+config MTK_IOMMU
+	bool "MTK IOMMU Support"
+	select IOMMU_API
+	select IOMMU_DMA
+	select MTK_SMI
+	help
+	  Support for the IOMMUs on certain Mediatek SOCs.
+	  These IOMMUs allow the multimedia hardware access discontinuous memory.
+
+	  If unsure, say N here.
+
 endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 37bfc4e..f2a8027 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
 obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
 obj-$(CONFIG_IOMMU_IOVA) += iova.o
 obj-$(CONFIG_OF_IOMMU)	+= of_iommu.o
+obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o mtk_iommu_pagetable.o
 obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
 obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
 obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
new file mode 100644
index 0000000..d62d4ab
--- /dev/null
+++ b/drivers/iommu/mtk_iommu.c
@@ -0,0 +1,754 @@
+/*
+ * Copyright (c) 2014-2015 MediaTek Inc.
+ * Author: Yong Wu <yong.wu at mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "m4u:"fmt
+
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/iommu.h>
+#include <linux/errno.h>
+#include <linux/list.h>
+#include <linux/memblock.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-iommu.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/mtk-smi.h>
+#include <asm/cacheflush.h>
+
+#include "mtk_iommu.h"
+
+#define REG_MMUG_PT_BASE	 0x0
+
+#define REG_MMU_INVLD		 0x20
+#define F_MMU_INV_ALL		 0x2
+#define F_MMU_INV_RANGE		 0x1
+
+#define REG_MMU_INVLD_SA	 0x24
+#define REG_MMU_INVLD_EA         0x28
+
+#define REG_MMU_INVLD_SEC         0x2c
+#define F_MMU_INV_SEC_ALL          0x2
+#define F_MMU_INV_SEC_RANGE        0x1
+
+#define REG_INVLID_SEL	         0x38
+#define F_MMU_INV_EN_L1		 BIT(0)
+#define F_MMU_INV_EN_L2		 BIT(1)
+
+#define REG_MMU_STANDARD_AXI_MODE   0x48
+#define REG_MMU_DCM_DIS             0x50
+#define REG_MMU_LEGACY_4KB_MODE     0x60
+
+#define REG_MMU_CTRL_REG                 0x110
+#define F_MMU_CTRL_REROUTE_PFQ_TO_MQ_EN  BIT(4)
+#define F_MMU_CTRL_TF_PROT_VAL(prot)      (((prot) & 0x3)<<5)
+#define F_MMU_CTRL_COHERE_EN             BIT(8)
+
+#define REG_MMU_IVRP_PADDR               0x114
+#define F_MMU_IVRP_PA_SET(PA)            (PA>>1)
+
+#define REG_MMU_INT_L2_CONTROL      0x120
+#define F_INT_L2_CLR_BIT            BIT(12)
+
+#define REG_MMU_INT_MAIN_CONTROL    0x124
+#define F_INT_TRANSLATION_FAULT(MMU)           (1<<(0+(((MMU)<<1)|((MMU)<<2))))
+#define F_INT_MAIN_MULTI_HIT_FAULT(MMU)        (1<<(1+(((MMU)<<1)|((MMU)<<2))))
+#define F_INT_INVALID_PA_FAULT(MMU)            (1<<(2+(((MMU)<<1)|((MMU)<<2))))
+#define F_INT_ENTRY_REPLACEMENT_FAULT(MMU)     (1<<(3+(((MMU)<<1)|((MMU)<<2))))
+#define F_INT_TLB_MISS_FAULT(MMU)              (1<<(4+(((MMU)<<1)|((MMU)<<2))))
+#define F_INT_PFH_FIFO_ERR(MMU)                (1<<(6+(((MMU)<<1)|((MMU)<<2))))
+
+#define REG_MMU_CPE_DONE              0x12C
+
+#define REG_MMU_MAIN_FAULT_ST         0x134
+
+#define REG_MMU_FAULT_VA(mmu)         (0x13c+((mmu)<<3))
+#define F_MMU_FAULT_VA_MSK            ((~0x0)<<12)
+#define F_MMU_FAULT_VA_WRITE_BIT       BIT(1)
+#define F_MMU_FAULT_VA_LAYER_BIT       BIT(0)
+
+#define REG_MMU_INVLD_PA(mmu)         (0x140+((mmu)<<3))
+#define REG_MMU_INT_ID(mmu)           (0x150+((mmu)<<2))
+#define F_MMU0_INT_ID_TF_MSK          (~0x3)	/* only for MM iommu. */
+
+#define MTK_TFID(larbid, portid) ((larbid << 7) | (portid << 2))
+
+static const struct mtk_iommu_port mtk_iommu_mt8173_port[] = {
+	/* port name                m4uid slaveid larbid portid tfid */
+	/* larb0 */
+	{"M4U_PORT_DISP_OVL0",          0,  0,    0,  0, MTK_TFID(0, 0)},
+	{"M4U_PORT_DISP_RDMA0",         0,  0,    0,  1, MTK_TFID(0, 1)},
+	{"M4U_PORT_DISP_WDMA0",         0,  0,    0,  2, MTK_TFID(0, 2)},
+	{"M4U_PORT_DISP_OD_R",          0,  0,    0,  3, MTK_TFID(0, 3)},
+	{"M4U_PORT_DISP_OD_W",          0,  0,    0,  4, MTK_TFID(0, 4)},
+	{"M4U_PORT_MDP_RDMA0",          0,  0,    0,  5, MTK_TFID(0, 5)},
+	{"M4U_PORT_MDP_WDMA",           0,  0,    0,  6, MTK_TFID(0, 6)},
+	{"M4U_PORT_MDP_WROT0",          0,  0,    0,  7, MTK_TFID(0, 7)},
+
+	/* larb1 */
+	{"M4U_PORT_HW_VDEC_MC_EXT",      0,  0,   1,  0, MTK_TFID(1, 0)},
+	{"M4U_PORT_HW_VDEC_PP_EXT",      0,  0,   1,  1, MTK_TFID(1, 1)},
+	{"M4U_PORT_HW_VDEC_UFO_EXT",     0,  0,   1,  2, MTK_TFID(1, 2)},
+	{"M4U_PORT_HW_VDEC_VLD_EXT",     0,  0,   1,  3, MTK_TFID(1, 3)},
+	{"M4U_PORT_HW_VDEC_VLD2_EXT",    0,  0,   1,  4, MTK_TFID(1, 4)},
+	{"M4U_PORT_HW_VDEC_AVC_MV_EXT",  0,  0,   1,  5, MTK_TFID(1, 5)},
+	{"M4U_PORT_HW_VDEC_PRED_RD_EXT", 0,  0,   1,  6, MTK_TFID(1, 6)},
+	{"M4U_PORT_HW_VDEC_PRED_WR_EXT", 0,  0,   1,  7, MTK_TFID(1, 7)},
+	{"M4U_PORT_HW_VDEC_PPWRAP_EXT",  0,  0,   1,  8, MTK_TFID(1, 8)},
+
+	/* larb2 */
+	{"M4U_PORT_IMGO",                0,  0,    2,  0, MTK_TFID(2, 0)},
+	{"M4U_PORT_RRZO",                0,  0,    2,  1, MTK_TFID(2, 1)},
+	{"M4U_PORT_AAO",                 0,  0,    2,  2, MTK_TFID(2, 2)},
+	{"M4U_PORT_LCSO",                0,  0,    2,  3, MTK_TFID(2, 3)},
+	{"M4U_PORT_ESFKO",               0,  0,    2,  4, MTK_TFID(2, 4)},
+	{"M4U_PORT_IMGO_D",              0,  0,    2,  5, MTK_TFID(2, 5)},
+	{"M4U_PORT_LSCI",                0,  0,    2,  6, MTK_TFID(2, 6)},
+	{"M4U_PORT_LSCI_D",              0,  0,    2,  7, MTK_TFID(2, 7)},
+	{"M4U_PORT_BPCI",                0,  0,    2,  8, MTK_TFID(2, 8)},
+	{"M4U_PORT_BPCI_D",              0,  0,    2,  9, MTK_TFID(2, 9)},
+	{"M4U_PORT_UFDI",                0,  0,    2,  10, MTK_TFID(2, 10)},
+	{"M4U_PORT_IMGI",                0,  0,    2,  11, MTK_TFID(2, 11)},
+	{"M4U_PORT_IMG2O",               0,  0,    2,  12, MTK_TFID(2, 12)},
+	{"M4U_PORT_IMG3O",               0,  0,    2,  13, MTK_TFID(2, 13)},
+	{"M4U_PORT_VIPI",                0,  0,    2,  14, MTK_TFID(2, 14)},
+	{"M4U_PORT_VIP2I",               0,  0,    2,  15, MTK_TFID(2, 15)},
+	{"M4U_PORT_VIP3I",               0,  0,    2,  16, MTK_TFID(2, 16)},
+	{"M4U_PORT_LCEI",                0,  0,    2,  17, MTK_TFID(2, 17)},
+	{"M4U_PORT_RB",                  0,  0,    2,  18, MTK_TFID(2, 18)},
+	{"M4U_PORT_RP",                  0,  0,    2,  19, MTK_TFID(2, 19)},
+	{"M4U_PORT_WR",                  0,  0,    2,  20, MTK_TFID(2, 20)},
+
+	/* larb3 */
+	{"M4U_PORT_VENC_RCPU",            0,  0,   3,  0, MTK_TFID(3, 0)},
+	{"M4U_PORT_VENC_REC",             0,  0,   3,  1, MTK_TFID(3, 1)},
+	{"M4U_PORT_VENC_BSDMA",           0,  0,   3,  2, MTK_TFID(3, 2)},
+	{"M4U_PORT_VENC_SV_COMV",         0,  0,   3,  3, MTK_TFID(3, 3)},
+	{"M4U_PORT_VENC_RD_COMV",         0,  0,   3,  4, MTK_TFID(3, 4)},
+	{"M4U_PORT_JPGENC_RDMA",          0,  0,   3,  5, MTK_TFID(3, 5)},
+	{"M4U_PORT_JPGENC_BSDMA",         0,  0,   3,  6, MTK_TFID(3, 6)},
+	{"M4U_PORT_JPGDEC_WDMA",          0,  0,   3,  7, MTK_TFID(3, 7)},
+	{"M4U_PORT_JPGDEC_BSDMA",         0,  0,   3,  8, MTK_TFID(3, 8)},
+	{"M4U_PORT_VENC_CUR_LUMA",        0,  0,   3,  9, MTK_TFID(3, 9)},
+	{"M4U_PORT_VENC_CUR_CHROMA",      0,  0,   3,  10, MTK_TFID(3, 10)},
+	{"M4U_PORT_VENC_REF_LUMA",        0,  0,   3,  11, MTK_TFID(3, 11)},
+	{"M4U_PORT_VENC_REF_CHROMA",      0,  0,   3,  12, MTK_TFID(3, 12)},
+	{"M4U_PORT_VENC_NBM_RDMA",        0,  0,   3,  13, MTK_TFID(3, 13)},
+	{"M4U_PORT_VENC_NBM_WDMA",        0,  0,   3,  14, MTK_TFID(3, 14)},
+
+	/* larb4 */
+	{"M4U_PORT_DISP_OVL1",             0,  0,   4,  0, MTK_TFID(4, 0)},
+	{"M4U_PORT_DISP_RDMA1",            0,  0,   4,  1, MTK_TFID(4, 1)},
+	{"M4U_PORT_DISP_RDMA2",            0,  0,   4,  2, MTK_TFID(4, 2)},
+	{"M4U_PORT_DISP_WDMA1",            0,  0,   4,  3, MTK_TFID(4, 3)},
+	{"M4U_PORT_MDP_RDMA1",             0,  0,   4,  4, MTK_TFID(4, 4)},
+	{"M4U_PORT_MDP_WROT1",             0,  0,   4,  5, MTK_TFID(4, 5)},
+
+	/* larb5 */
+	{"M4U_PORT_VENC_RCPU_SET2",        0,  0,    5,  0, MTK_TFID(5, 0)},
+	{"M4U_PORT_VENC_REC_FRM_SET2",     0,  0,    5,  1, MTK_TFID(5, 1)},
+	{"M4U_PORT_VENC_REF_LUMA_SET2",    0,  0,    5,  2, MTK_TFID(5, 2)},
+	{"M4U_PORT_VENC_REC_CHROMA_SET2",  0,  0,    5,  3, MTK_TFID(5, 3)},
+	{"M4U_PORT_VENC_BSDMA_SET2",       0,  0,    5,  4, MTK_TFID(5, 4)},
+	{"M4U_PORT_VENC_CUR_LUMA_SET2",    0,  0,    5,  5, MTK_TFID(5, 5)},
+	{"M4U_PORT_VENC_CUR_CHROMA_SET2",  0,  0,    5,  6, MTK_TFID(5, 6)},
+	{"M4U_PORT_VENC_RD_COMA_SET2",     0,  0,    5,  7, MTK_TFID(5, 7)},
+	{"M4U_PORT_VENC_SV_COMA_SET2",     0,  0,    5,  8, MTK_TFID(5, 8)},
+
+	/* perisys iommu */
+	{"M4U_PORT_RESERVE",               1,  0,    6,  0, 0xff},
+	{"M4U_PORT_SPM",                   1,  0,    6,  1, 0x50},
+	{"M4U_PORT_MD32",                  1,  0,    6,  2, 0x90},
+	{"M4U_PORT_PTP_THERM",             1,  0,    6,  4, 0xd0},
+	{"M4U_PORT_PWM",                   1,  0,    6,  5, 0x1},
+	{"M4U_PORT_MSDC1",                 1,  0,    6,  6, 0x21},
+	{"M4U_PORT_MSDC2",                 1,  0,    6,  7, 0x41},
+	{"M4U_PORT_NFI",                   1,  0,    6,  8, 0x8},
+	{"M4U_PORT_AUDIO",                 1,  0,    6,  9, 0x48},
+	{"M4U_PORT_RESERVED2",             1,  0,    6,  10, 0xfe},
+	{"M4U_PORT_HSIC_XHCI",             1,  0,    6,  11, 0x9},
+
+	{"M4U_PORT_HSIC_MAS",              1,  0,    6,  12, 0x11},
+	{"M4U_PORT_HSIC_DEV",              1,  0,    6,  13, 0x19},
+	{"M4U_PORT_AP_DMA",                1,  0,    6,  14, 0x18},
+	{"M4U_PORT_HSIC_DMA",              1,  0,    6,  15, 0xc8},
+	{"M4U_PORT_MSDC0",                 1,  0,    6,  16, 0x0},
+	{"M4U_PORT_MSDC3",                 1,  0,    6,  17, 0x20},
+	{"M4U_PORT_UNKNOWN",               1,  0,    6,  18, 0xf},
+};
+
+static const struct mtk_iommu_cfg mtk_iommu_mt8173_cfg = {
+	.larb_nr = 6,
+	.m4u_port_nr = ARRAY_SIZE(mtk_iommu_mt8173_port),
+	.pport = mtk_iommu_mt8173_port,
+};
+
+static const char *mtk_iommu_get_port_name(const struct mtk_iommu_info *piommu,
+					   unsigned int portid)
+{
+	const struct mtk_iommu_port *pcurport = NULL;
+
+	pcurport = piommu->imucfg->pport + portid;
+	if (portid < piommu->imucfg->m4u_port_nr && pcurport)
+		return pcurport->port_name;
+	else
+		return "UNKNOWN_PORT";
+}
+
+static int mtk_iommu_get_port_by_tfid(const struct mtk_iommu_info *pimu,
+				      int tf_id)
+{
+	const struct mtk_iommu_cfg *pimucfg = pimu->imucfg;
+	int i;
+	unsigned int portid = pimucfg->m4u_port_nr;
+
+	for (i = 0; i < pimucfg->m4u_port_nr; i++) {
+		if (pimucfg->pport[i].tf_id == tf_id) {
+			portid = i;
+			break;
+		}
+	}
+	if (i == pimucfg->m4u_port_nr)
+		dev_err(pimu->dev, "tf_id find fail, tfid %d\n", tf_id);
+	return portid;
+}
+
+static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
+{
+	struct iommu_domain *domain = dev_id;
+	struct mtk_iommu_domain *mtkdomain = domain->priv;
+	struct mtk_iommu_info *piommu = mtkdomain->piommuinfo;
+
+	if (irq == piommu->irq)
+		report_iommu_fault(domain, piommu->dev, 0, 0);
+	else
+		dev_err(piommu->dev, "irq number:%d\n", irq);
+
+	return IRQ_HANDLED;
+}
+
+static inline void mtk_iommu_clear_intr(void __iomem *m4u_base)
+{
+	u32 val;
+
+	val = readl(m4u_base + REG_MMU_INT_L2_CONTROL);
+	val |= F_INT_L2_CLR_BIT;
+	writel(val, m4u_base + REG_MMU_INT_L2_CONTROL);
+}
+
+static int mtk_iommu_invalidate_tlb(const struct mtk_iommu_info *piommu,
+				    int isinvall, unsigned int iova_start,
+				    unsigned int iova_end)
+{
+	void __iomem *m4u_base = piommu->m4u_base;
+	u32 val;
+	u64 start, end;
+
+	start = sched_clock();
+
+	if (!isinvall) {
+		iova_start = round_down(iova_start, SZ_4K);
+		iova_end = round_up(iova_end, SZ_4K);
+	}
+
+	val = F_MMU_INV_EN_L2 | F_MMU_INV_EN_L1;
+
+	writel(val, m4u_base + REG_INVLID_SEL);
+
+	if (isinvall) {
+		writel(F_MMU_INV_ALL, m4u_base + REG_MMU_INVLD);
+	} else {
+		writel(iova_start, m4u_base + REG_MMU_INVLD_SA);
+		writel(iova_end, m4u_base + REG_MMU_INVLD_EA);
+		writel(F_MMU_INV_RANGE, m4u_base + REG_MMU_INVLD);
+
+		while (!readl(m4u_base + REG_MMU_CPE_DONE)) {
+			end = sched_clock();
+			if (end - start >= 100000000ULL) {
+				dev_warn(piommu->dev, "invalid don't done\n");
+				writel(F_MMU_INV_ALL, m4u_base + REG_MMU_INVLD);
+			}
+		};
+		writel(0, m4u_base + REG_MMU_CPE_DONE);
+	}
+
+	return 0;
+}
+
+static int mtk_iommu_fault_handler(struct iommu_domain *imudomain,
+				   struct device *dev, unsigned long iova,
+				   int m4uindex, void *pimu)
+{
+	void __iomem *m4u_base;
+	u32 int_state, regval;
+	int m4u_slave_id = 0;
+	unsigned int layer, write, m4u_port;
+	unsigned int fault_mva, fault_pa;
+	struct mtk_iommu_info *piommu = pimu;
+	struct mtk_iommu_domain *mtkdomain = imudomain->priv;
+
+	m4u_base = piommu->m4u_base;
+	int_state = readl(m4u_base + REG_MMU_MAIN_FAULT_ST);
+
+	/* read error info from registers */
+	fault_mva = readl(m4u_base + REG_MMU_FAULT_VA(m4u_slave_id));
+	layer = !!(fault_mva & F_MMU_FAULT_VA_LAYER_BIT);
+	write = !!(fault_mva & F_MMU_FAULT_VA_WRITE_BIT);
+	fault_mva &= F_MMU_FAULT_VA_MSK;
+	fault_pa = readl(m4u_base + REG_MMU_INVLD_PA(m4u_slave_id));
+	regval = readl(m4u_base + REG_MMU_INT_ID(m4u_slave_id));
+	regval &= F_MMU0_INT_ID_TF_MSK;
+	m4u_port = mtk_iommu_get_port_by_tfid(piommu, regval);
+
+	if (int_state & F_INT_TRANSLATION_FAULT(m4u_slave_id)) {
+		struct m4u_pte_info_t pte;
+		unsigned long flags;
+
+		spin_lock_irqsave(&mtkdomain->pgtlock, flags);
+		m4u_get_pte_info(mtkdomain, fault_mva, &pte);
+		spin_unlock_irqrestore(&mtkdomain->pgtlock, flags);
+
+		if (pte.size == MMU_SMALL_PAGE_SIZE ||
+		    pte.size == MMU_LARGE_PAGE_SIZE) {
+			dev_err_ratelimited(
+				dev,
+				"fault:port=%s iova=0x%x pa=0x%x layer=%d %s;"
+				"pgd(0x%x)->pte(0x%x)->pa(%pad)sz(0x%x)Valid(%d)\n",
+				mtk_iommu_get_port_name(piommu, m4u_port),
+				fault_mva, fault_pa, layer,
+				write ? "write" : "read",
+				imu_pgd_val(*pte.pgd), imu_pte_val(*pte.pte),
+				&pte.pa, pte.size, pte.valid);
+		} else {
+			dev_err_ratelimited(
+				dev,
+				"fault:port=%s iova=0x%x pa=0x%x layer=%d %s;"
+				"pgd(0x%x)->pa(%pad)sz(0x%x)Valid(%d)\n",
+				mtk_iommu_get_port_name(piommu, m4u_port),
+				fault_mva, fault_pa, layer,
+				write ? "write" : "read",
+				imu_pgd_val(*pte.pgd),
+				&pte.pa, pte.size, pte.valid);
+		}
+	}
+
+	if (int_state & F_INT_MAIN_MULTI_HIT_FAULT(m4u_slave_id))
+		dev_err_ratelimited(dev, "multi-hit!port=%s iova=0x%x\n",
+				    mtk_iommu_get_port_name(piommu, m4u_port),
+				    fault_mva);
+
+	if (int_state & F_INT_INVALID_PA_FAULT(m4u_slave_id)) {
+		if (!(int_state & F_INT_TRANSLATION_FAULT(m4u_slave_id)))
+			dev_err_ratelimited(dev, "invalid pa!port=%s iova=0x%x\n",
+					    mtk_iommu_get_port_name(piommu,
+								    m4u_port),
+					    fault_mva);
+	}
+	if (int_state & F_INT_ENTRY_REPLACEMENT_FAULT(m4u_slave_id))
+		dev_err_ratelimited(dev, "replace-fault!port=%s iova=0x%x\n",
+				    mtk_iommu_get_port_name(piommu, m4u_port),
+				    fault_mva);
+
+	if (int_state & F_INT_TLB_MISS_FAULT(m4u_slave_id))
+		dev_err_ratelimited(dev, "tlb miss-fault!port=%s iova=0x%x\n",
+				    mtk_iommu_get_port_name(piommu, m4u_port),
+				    fault_mva);
+
+	mtk_iommu_invalidate_tlb(piommu, 1, 0, 0);
+
+	mtk_iommu_clear_intr(m4u_base);
+
+	return 0;
+}
+
+static int mtk_iommu_parse_dt(struct platform_device *pdev,
+			      struct mtk_iommu_info *piommu)
+{
+	struct device *piommudev = &pdev->dev;
+	struct device_node *ofnode;
+	struct resource *res;
+	unsigned int mtk_iommu_cell = 0;
+	unsigned int i;
+
+	ofnode = piommudev->of_node;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	piommu->m4u_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(piommu->m4u_base)) {
+		dev_err(piommudev, "m4u_base %p err\n", piommu->m4u_base);
+		goto iommu_dts_err;
+	}
+
+	piommu->irq = platform_get_irq(pdev, 0);
+	if (piommu->irq < 0) {
+		dev_err(piommudev, "irq err %d\n", piommu->irq);
+		goto iommu_dts_err;
+	}
+
+	piommu->m4u_infra_clk = devm_clk_get(piommudev, "infra_m4u");
+	if (IS_ERR(piommu->m4u_infra_clk)) {
+		dev_err(piommudev, "clk err %p\n", piommu->m4u_infra_clk);
+		goto iommu_dts_err;
+	}
+
+	of_property_read_u32(ofnode, "#iommu-cells", &mtk_iommu_cell);
+	if (mtk_iommu_cell != 1) {
+		dev_err(piommudev, "iommu-cell fail:%d\n", mtk_iommu_cell);
+		goto iommu_dts_err;
+	}
+
+	for (i = 0; i < piommu->imucfg->larb_nr; i++) {
+		struct device_node *larbnode;
+
+		larbnode = of_parse_phandle(ofnode, "larb", i);
+		piommu->larbpdev[i] = of_find_device_by_node(larbnode);
+		of_node_put(larbnode);
+		if (!piommu->larbpdev[i]) {
+			dev_err(piommudev, "larb pdev fail at larb%d\n", i);
+			goto iommu_dts_err;
+		}
+	}
+
+	return 0;
+
+iommu_dts_err:
+	return -EINVAL;
+}
+
+static int mtk_iommu_hw_init(const struct mtk_iommu_domain *mtkdomain)
+{
+	struct mtk_iommu_info *piommu = mtkdomain->piommuinfo;
+	void __iomem *gm4ubaseaddr = piommu->m4u_base;
+	phys_addr_t protectpa;
+	u32 regval, protectreg;
+	int ret = 0;
+
+	ret = clk_prepare_enable(piommu->m4u_infra_clk);
+	if (ret) {
+		dev_err(piommu->dev, "m4u clk enable error\n");
+		return -ENODEV;
+	}
+
+	writel((u32)mtkdomain->pgd_pa, gm4ubaseaddr + REG_MMUG_PT_BASE);
+
+	regval = F_MMU_CTRL_REROUTE_PFQ_TO_MQ_EN |
+		F_MMU_CTRL_TF_PROT_VAL(2) |
+		F_MMU_CTRL_COHERE_EN;
+	writel(regval, gm4ubaseaddr + REG_MMU_CTRL_REG);
+
+	writel(0x6f, gm4ubaseaddr + REG_MMU_INT_L2_CONTROL);
+	writel(0xffffffff, gm4ubaseaddr + REG_MMU_INT_MAIN_CONTROL);
+
+	/* protect memory,HW will write here while translation fault */
+	protectpa = __virt_to_phys(piommu->protect_va);
+	protectpa = ALIGN(protectpa, MTK_PROTECT_PA_ALIGN);
+	protectreg = (u32)F_MMU_IVRP_PA_SET(protectpa);
+	writel(protectreg, gm4ubaseaddr + REG_MMU_IVRP_PADDR);
+
+	writel(0, gm4ubaseaddr + REG_MMU_DCM_DIS);
+	writel(0, gm4ubaseaddr + REG_MMU_STANDARD_AXI_MODE);
+
+	return 0;
+}
+
+static inline void mtk_iommu_config_port(struct mtk_iommu_info *piommu,
+					 int portid)
+{
+	int larb, larb_port;
+
+	larb = piommu->imucfg->pport[portid].larb_id;
+	larb_port = piommu->imucfg->pport[portid].port_id;
+
+	mtk_smi_config_port(piommu->larbpdev[larb], larb_port);
+}
+
+/*
+ * pimudev is a global var for dma_alloc_coherent.
+ * It is not accepatable, we will delete it if "domain_alloc" is enabled
+ */
+static struct device *pimudev;
+
+static int mtk_iommu_domain_init(struct iommu_domain *domain)
+{
+	struct mtk_iommu_domain *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->pgd = dma_alloc_coherent(pimudev, M4U_PGD_SIZE, &priv->pgd_pa,
+				       GFP_KERNEL);
+	if (!priv->pgd) {
+		pr_err("dma_alloc_coherent pagetable fail\n");
+		goto err_pgtable;
+	}
+
+	if (!IS_ALIGNED(priv->pgd_pa, M4U_PGD_SIZE)) {
+		pr_err("pagetable not aligned pa 0x%pad-0x%p align 0x%x\n",
+		       &priv->pgd_pa, priv->pgd, M4U_PGD_SIZE);
+		goto err_pgtable;
+	}
+
+	memset(priv->pgd, 0, M4U_PGD_SIZE);
+
+	spin_lock_init(&priv->pgtlock);
+	spin_lock_init(&priv->portlock);
+	domain->priv = priv;
+
+	domain->geometry.aperture_start = 0;
+	domain->geometry.aperture_end   = (unsigned int)~0;
+	domain->geometry.force_aperture = true;
+
+	return 0;
+
+err_pgtable:
+	if (priv->pgd)
+		dma_free_coherent(pimudev, M4U_PGD_SIZE, priv->pgd,
+				  priv->pgd_pa);
+	kfree(priv);
+	return -ENOMEM;
+}
+
+static void mtk_iommu_domain_destroy(struct iommu_domain *domain)
+{
+	struct mtk_iommu_domain *priv = domain->priv;
+
+	dma_free_coherent(priv->piommuinfo->dev, M4U_PGD_SIZE,
+			  priv->pgd, priv->pgd_pa);
+	kfree(domain->priv);
+	domain->priv = NULL;
+}
+
+static int mtk_iommu_attach_device(struct iommu_domain *domain,
+				   struct device *dev)
+{
+	unsigned long flags;
+	struct mtk_iommu_domain *priv = domain->priv;
+	struct mtk_iommu_info *piommu = priv->piommuinfo;
+	struct of_phandle_args out_args = {0};
+	struct device *imudev;
+	unsigned int i = 0;
+
+	if (!piommu)
+		goto imudev;
+	else
+		imudev = piommu->dev;
+
+	spin_lock_irqsave(&priv->portlock, flags);
+
+	while (!of_parse_phandle_with_args(dev->of_node, "iommus",
+					   "#iommu-cells", i, &out_args)) {
+		if (1 == out_args.args_count) {
+			unsigned int portid = out_args.args[0];
+
+			dev_dbg(dev, "iommu add port:%d\n", portid);
+
+			mtk_iommu_config_port(piommu, portid);
+
+			if (i == 0)
+				dev->archdata.dma_ops =
+					piommu->dev->archdata.dma_ops;
+		}
+		i++;
+	}
+
+	spin_unlock_irqrestore(&priv->portlock, flags);
+
+imudev:
+	return 0;
+}
+
+static void mtk_iommu_detach_device(struct iommu_domain *domain,
+				    struct device *dev)
+{
+}
+
+static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
+			 phys_addr_t paddr, size_t size, int prot)
+{
+	struct mtk_iommu_domain *priv = domain->priv;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&priv->pgtlock, flags);
+	ret = m4u_map(priv, (unsigned int)iova, paddr, size, prot);
+	mtk_iommu_invalidate_tlb(priv->piommuinfo, 0,
+				 iova, iova + size - 1);
+	spin_unlock_irqrestore(&priv->pgtlock, flags);
+
+	return ret;
+}
+
+static size_t mtk_iommu_unmap(struct iommu_domain *domain,
+			      unsigned long iova, size_t size)
+{
+	struct mtk_iommu_domain *priv = domain->priv;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->pgtlock, flags);
+	m4u_unmap(priv, (unsigned int)iova, size);
+	mtk_iommu_invalidate_tlb(priv->piommuinfo, 0,
+				 iova, iova + size - 1);
+	spin_unlock_irqrestore(&priv->pgtlock, flags);
+
+	return size;
+}
+
+static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
+					  dma_addr_t iova)
+{
+	struct mtk_iommu_domain *priv = domain->priv;
+	unsigned long flags;
+	struct m4u_pte_info_t pte;
+
+	spin_lock_irqsave(&priv->pgtlock, flags);
+	m4u_get_pte_info(priv, (unsigned int)iova, &pte);
+	spin_unlock_irqrestore(&priv->pgtlock, flags);
+
+	return pte.pa;
+}
+
+static struct iommu_ops mtk_iommu_ops = {
+	.domain_init = mtk_iommu_domain_init,
+	.domain_destroy = mtk_iommu_domain_destroy,
+	.attach_dev = mtk_iommu_attach_device,
+	.detach_dev = mtk_iommu_detach_device,
+	.map = mtk_iommu_map,
+	.unmap = mtk_iommu_unmap,
+	.map_sg = default_iommu_map_sg,
+	.iova_to_phys = mtk_iommu_iova_to_phys,
+	.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static const struct of_device_id mtk_iommu_of_ids[] = {
+	{ .compatible = "mediatek,mt8173-iommu",
+	  .data = &mtk_iommu_mt8173_cfg,
+	},
+	{}
+};
+
+static int mtk_iommu_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct iommu_domain *domain;
+	struct mtk_iommu_domain *mtk_domain;
+	struct mtk_iommu_info *piommu;
+	struct iommu_dma_domain  *dom;
+	const struct of_device_id *of_id;
+
+	piommu = devm_kzalloc(&pdev->dev, sizeof(struct mtk_iommu_info),
+			      GFP_KERNEL);
+	if (!piommu)
+		return -ENOMEM;
+
+	pimudev = &pdev->dev;
+	piommu->dev = &pdev->dev;
+
+	of_id = of_match_node(mtk_iommu_of_ids, pdev->dev.of_node);
+	if (!of_id)
+		return -ENODEV;
+
+	piommu->protect_va = devm_kmalloc(piommu->dev, MTK_PROTECT_PA_ALIGN*2,
+					  GFP_KERNEL);
+	if (!piommu->protect_va)
+		goto protect_err;
+	memset(piommu->protect_va, 0x55, MTK_PROTECT_PA_ALIGN*2);
+
+	piommu->imucfg = (const struct mtk_iommu_cfg *)of_id->data;
+
+	ret = mtk_iommu_parse_dt(pdev, piommu);
+	if (ret) {
+		dev_err(piommu->dev, "iommu dt parse fail\n");
+		goto protect_err;
+	}
+
+	/* alloc memcache for level-2 pgt */
+	piommu->m4u_pte_kmem = kmem_cache_create("m4u_pte", IMU_BYTES_PER_PTE,
+						 IMU_BYTES_PER_PTE, 0, NULL);
+
+	if (IS_ERR_OR_NULL(piommu->m4u_pte_kmem)) {
+		dev_err(piommu->dev, "pte cached create fail %p\n",
+			piommu->m4u_pte_kmem);
+		goto protect_err;
+	}
+
+	arch_setup_dma_ops(piommu->dev, 0, (1ULL<<32) - 1, &mtk_iommu_ops, 0);
+
+	dom = get_dma_domain(piommu->dev);
+	domain = iommu_dma_raw_domain(dom);
+
+	mtk_domain = domain->priv;
+	mtk_domain->piommuinfo = piommu;
+
+	if (!domain)
+		goto pte_err;
+
+	ret = mtk_iommu_hw_init(mtk_domain);
+	if (ret < 0)
+		goto hw_err;
+
+	if (devm_request_irq(piommu->dev, piommu->irq,
+			     mtk_iommu_isr, IRQF_TRIGGER_NONE,
+			     "mtkiommu", (void *)domain)) {
+		dev_err(piommu->dev, "IRQ request %d failed\n",
+			piommu->irq);
+		goto hw_err;
+	}
+
+	iommu_set_fault_handler(domain, mtk_iommu_fault_handler, piommu);
+
+	dev_set_drvdata(piommu->dev, piommu);
+
+	return 0;
+hw_err:
+	arch_teardown_dma_ops(piommu->dev);
+pte_err:
+	kmem_cache_destroy(piommu->m4u_pte_kmem);
+protect_err:
+	dev_err(piommu->dev, "probe error\n");
+	return 0;
+}
+
+static int mtk_iommu_remove(struct platform_device *pdev)
+{
+	struct mtk_iommu_info *piommu =	dev_get_drvdata(&pdev->dev);
+
+	arch_teardown_dma_ops(piommu->dev);
+	kmem_cache_destroy(piommu->m4u_pte_kmem);
+
+	return 0;
+}
+
+static struct platform_driver mtk_iommu_driver = {
+	.probe	= mtk_iommu_probe,
+	.remove	= mtk_iommu_remove,
+	.driver	= {
+		.name = "mtkiommu",
+		.of_match_table = mtk_iommu_of_ids,
+	}
+};
+
+static int __init mtk_iommu_init(void)
+{
+	return platform_driver_register(&mtk_iommu_driver);
+}
+
+subsys_initcall(mtk_iommu_init);
+
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
new file mode 100644
index 0000000..239471f
--- /dev/null
+++ b/drivers/iommu/mtk_iommu.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2014-2015 MediaTek Inc.
+ * Author: Yong Wu <yong.wu at mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef MTK_IOMMU_PLATFORM_H
+#define MTK_IOMMU_PLATFORM_H
+
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/platform_device.h>
+
+#include "mtk_iommu_pagetable.h"
+
+#define M4U_PGD_SIZE       SZ_16K   /* pagetable size,mt8173 */
+
+#define MTK_PROTECT_PA_ALIGN  128
+
+#define MTK_IOMMU_LARB_MAX_NR 8
+#define MTK_IOMMU_PORT_MAX_NR 100
+
+struct mtk_iommu_port {
+	const char *port_name;
+	unsigned int m4u_id:2;
+	unsigned int m4u_slave:2;/* main tlb index in mm iommu */
+	unsigned int larb_id:4;
+	unsigned int port_id:8;/* port id in larb */
+	unsigned int tf_id:16; /* translation fault id */
+};
+
+struct mtk_iommu_cfg {
+	unsigned int larb_nr;
+	unsigned int m4u_port_nr;
+	const struct mtk_iommu_port *pport;
+};
+
+struct mtk_iommu_info {
+	void __iomem *m4u_base;
+	unsigned int  irq;
+	struct platform_device *larbpdev[MTK_IOMMU_LARB_MAX_NR];
+	struct clk *m4u_infra_clk;
+	void __iomem *protect_va;
+	struct device *dev;
+	struct kmem_cache *m4u_pte_kmem;
+	const struct mtk_iommu_cfg *imucfg;
+};
+
+struct mtk_iommu_domain {
+	struct imu_pgd_t *pgd;
+	dma_addr_t pgd_pa;
+	spinlock_t pgtlock;	/* lock for modifying page table */
+	spinlock_t portlock;    /* lock for config port */
+	struct mtk_iommu_info *piommuinfo;
+};
+
+int m4u_map(struct mtk_iommu_domain *m4u_domain, unsigned int iova,
+	    phys_addr_t paddr, unsigned int size, unsigned int prot);
+int m4u_unmap(struct mtk_iommu_domain *domain, unsigned int iova,
+	      unsigned int size);
+int m4u_get_pte_info(const struct mtk_iommu_domain *domain,
+		     unsigned int iova, struct m4u_pte_info_t *pte_info);
+
+#endif
diff --git a/drivers/iommu/mtk_iommu_pagetable.c b/drivers/iommu/mtk_iommu_pagetable.c
new file mode 100644
index 0000000..5fe9640
--- /dev/null
+++ b/drivers/iommu/mtk_iommu_pagetable.c
@@ -0,0 +1,439 @@
+/*
+ * Copyright (c) 2014-2015 MediaTek Inc.
+ * Author: Yong Wu <yong.wu at mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/iommu.h>
+#include <linux/errno.h>
+#include "asm/cacheflush.h"
+
+#include "mtk_iommu.h"
+#include "mtk_iommu_pagetable.h"
+
+/* 2 level pagetable: pgd -> pte */
+#define F_PTE_TYPE_GET(regval)  (regval & 0x3)
+#define F_PTE_TYPE_LARGE         BIT(0)
+#define F_PTE_TYPE_SMALL         BIT(1)
+#define F_PTE_B_BIT              BIT(2)
+#define F_PTE_C_BIT              BIT(3)
+#define F_PTE_BIT32_BIT          BIT(9)
+#define F_PTE_S_BIT              BIT(10)
+#define F_PTE_NG_BIT             BIT(11)
+#define F_PTE_PA_LARGE_MSK            (~0UL << 16)
+#define F_PTE_PA_LARGE_GET(regval)    ((regval >> 16) & 0xffff)
+#define F_PTE_PA_SMALL_MSK            (~0UL << 12)
+#define F_PTE_PA_SMALL_GET(regval)    ((regval >> 12) & (~0))
+#define F_PTE_TYPE_IS_LARGE_PAGE(pte) ((imu_pte_val(pte) & 0x3) == \
+					F_PTE_TYPE_LARGE)
+#define F_PTE_TYPE_IS_SMALL_PAGE(pte) ((imu_pte_val(pte) & 0x3) == \
+					F_PTE_TYPE_SMALL)
+
+#define F_PGD_TYPE_PAGE         (0x1)
+#define F_PGD_TYPE_PAGE_MSK     (0x3)
+#define F_PGD_TYPE_SECTION      (0x2)
+#define F_PGD_TYPE_SUPERSECTION   (0x2 | (1 << 18))
+#define F_PGD_TYPE_SECTION_MSK    (0x3 | (1 << 18))
+#define F_PGD_TYPE_IS_PAGE(pgd)   ((imu_pgd_val(pgd)&3) == 1)
+#define F_PGD_TYPE_IS_SECTION(pgd) \
+	(F_PGD_TYPE_IS_PAGE(pgd) ? 0 : \
+		((imu_pgd_val(pgd) & F_PGD_TYPE_SECTION_MSK) == \
+			F_PGD_TYPE_SECTION))
+#define F_PGD_TYPE_IS_SUPERSECTION(pgd) \
+	(F_PGD_TYPE_IS_PAGE(pgd) ? 0 : \
+		((imu_pgd_val(pgd) & F_PGD_TYPE_SECTION_MSK) ==\
+			F_PGD_TYPE_SUPERSECTION))
+
+#define F_PGD_B_BIT                     BIT(2)
+#define F_PGD_C_BIT                     BIT(3)
+#define F_PGD_BIT32_BIT                 BIT(9)
+#define F_PGD_S_BIT                     BIT(16)
+#define F_PGD_NG_BIT                    BIT(17)
+#define F_PGD_NS_BIT_PAGE(ns)           (ns << 3)
+#define F_PGD_NS_BIT_SECTION(ns)        (ns << 19)
+#define F_PGD_NS_BIT_SUPERSECTION(ns)   (ns << 19)
+
+#define imu_pgd_index(addr)		((addr) >> IMU_PGDIR_SHIFT)
+#define imu_pgd_offset(domain, addr) ((domain)->pgd + imu_pgd_index(addr))
+
+#define imu_pte_index(addr)    (((addr)>>IMU_PAGE_SHIFT)&(IMU_PTRS_PER_PTE - 1))
+#define imu_pte_offset_map(pgd, addr) (imu_pte_map(pgd) + imu_pte_index(addr))
+
+#define F_PGD_PA_PAGETABLE_MSK            (~0 << 10)
+#define F_PGD_PA_SECTION_MSK              (~0 << 20)
+#define F_PGD_PA_SUPERSECTION_MSK         (~0 << 24)
+
+static inline struct imu_pte_t *imu_pte_map(struct imu_pgd_t *pgd)
+{
+	unsigned int pte_pa = imu_pgd_val(*pgd);
+
+	return (struct imu_pte_t *)(__va(pte_pa
+					& F_PGD_PA_PAGETABLE_MSK));
+}
+
+static inline struct imu_pgd_t *imu_supersection_start(struct imu_pgd_t *pgd)
+{
+	return (struct imu_pgd_t *)(round_down((unsigned long)pgd, (16 * 4)));
+}
+
+static inline void m4u_set_pgd_val(struct imu_pgd_t *pgd, unsigned int val)
+{
+	imu_pgd_val(*pgd) = val;
+}
+
+static inline unsigned int __m4u_get_pgd_attr(unsigned int prot,
+					      bool super, bool imu4gmode)
+{
+	unsigned int pgprot;
+
+	pgprot = F_PGD_NS_BIT_SECTION(1) | F_PGD_S_BIT;
+	pgprot |= super ? F_PGD_TYPE_SUPERSECTION : F_PGD_TYPE_SECTION;
+	pgprot |= (prot & IOMMU_CACHE) ? (F_PGD_C_BIT | F_PGD_B_BIT) : 0;
+	pgprot |= imu4gmode ? F_PGD_BIT32_BIT : 0;
+
+	return pgprot;
+}
+
+static inline unsigned int __m4u_get_pte_attr(unsigned int prot,
+					      bool large, bool imu4gmode)
+{
+	unsigned int pgprot;
+
+	pgprot = F_PTE_S_BIT;
+	pgprot |= large ? F_PTE_TYPE_LARGE : F_PTE_TYPE_SMALL;
+	pgprot |= (prot & IOMMU_CACHE) ? (F_PGD_C_BIT | F_PGD_B_BIT) : 0;
+	pgprot |= imu4gmode ? F_PTE_BIT32_BIT : 0;
+
+	return pgprot;
+}
+
+static inline void m4u_pgtable_flush(void *vastart, void *vaend)
+{
+	/*
+	 * this function is not acceptable, we will use dma_map_single
+	 * or use dma_pool_create for the level2 pagetable.
+	 */
+	__dma_flush_range(vastart, vaend);
+}
+
+/* @return   0 -- pte is allocated
+ *     1 -- pte is not allocated, because it's allocated by others
+ *     <0 -- error
+ */
+static int m4u_alloc_pte(struct mtk_iommu_domain *domain, struct imu_pgd_t *pgd,
+			 unsigned int pgprot)
+{
+	void *pte_new_va;
+	phys_addr_t pte_new;
+	struct kmem_cache *pte_kmem = domain->piommuinfo->m4u_pte_kmem;
+	struct device *dev = domain->piommuinfo->dev;
+	unsigned int ret;
+
+	pte_new_va = kmem_cache_zalloc(pte_kmem, GFP_KERNEL);
+	if (unlikely(!pte_new_va)) {
+		dev_err(dev, "%s:fail, no memory\n", __func__);
+		return -ENOMEM;
+	}
+	pte_new = __virt_to_phys(pte_new_va);
+
+	/* check pte alignment -- must 1K align */
+	if (unlikely(pte_new & (IMU_BYTES_PER_PTE - 1))) {
+		dev_err(dev, "%s:fail, not align pa=0x%pa, va=0x%p\n",
+			__func__, &pte_new, pte_new_va);
+		kmem_cache_free(pte_kmem, (void *)pte_new_va);
+		return -ENOMEM;
+	}
+
+	/* because someone else may have allocated for this pgd first */
+	if (likely(!imu_pgd_val(*pgd))) {
+		m4u_set_pgd_val(pgd, (unsigned int)(pte_new) | pgprot);
+		dev_dbg(dev, "%s:pgd:0x%p,pte_va:0x%p,pte_pa:%pa,value:0x%x\n",
+			__func__, pgd, pte_new_va,
+			&pte_new, (unsigned int)(pte_new) | pgprot);
+		ret = 0;
+	} else {
+		/* allocated by other thread */
+		dev_dbg(dev, "m4u pte allocated by others: pgd=0x%p\n", pgd);
+		kmem_cache_free(pte_kmem, (void *)pte_new_va);
+		ret = 1;
+	}
+	return ret;
+}
+
+static int m4u_free_pte(struct mtk_iommu_domain *domain, struct imu_pgd_t *pgd)
+{
+	struct imu_pte_t *pte_old;
+	struct kmem_cache *pte_kmem = domain->piommuinfo->m4u_pte_kmem;
+
+	pte_old = imu_pte_map(pgd);
+	m4u_set_pgd_val(pgd, 0);
+
+	kmem_cache_free(pte_kmem, pte_old);
+
+	return 0;
+}
+
+static int m4u_map_page(struct mtk_iommu_domain *m4u_domain, unsigned int iova,
+			phys_addr_t pa, unsigned int prot, bool largepage)
+{
+	int ret;
+	struct imu_pgd_t *pgd;
+	struct imu_pte_t *pte;
+	unsigned int pte_new, pgprot;
+	unsigned int padscpt;
+	struct device *dev = m4u_domain->piommuinfo->dev;
+	unsigned int mask = largepage ?
+				F_PTE_PA_LARGE_MSK : F_PTE_PA_SMALL_MSK;
+	unsigned int i, ptenum = largepage ? 16 : 1;
+	bool imu4gmode = (pa > 0xffffffffL) ? true : false;
+
+	if ((iova & (~mask)) != ((unsigned int)pa & (~mask))) {
+		dev_err(dev, "error to mk_pte: iova=0x%x, pa=0x%pa, type=%s\n",
+			iova, &pa, largepage ? "large page" : "small page");
+		return -EINVAL;
+	}
+
+	iova &= mask;
+	padscpt = (unsigned int)pa & mask;
+
+	pgprot = F_PGD_TYPE_PAGE | F_PGD_NS_BIT_PAGE(1);
+	pgd = imu_pgd_offset(m4u_domain, iova);
+	if (!imu_pgd_val(*pgd)) {
+		ret = m4u_alloc_pte(m4u_domain, pgd, pgprot);
+		if (ret < 0)
+			return ret;
+		else if (ret > 0)
+			pte_new = 0;
+		else
+			pte_new = 1;
+	} else {
+		if ((imu_pgd_val(*pgd) & (~F_PGD_PA_PAGETABLE_MSK)) != pgprot) {
+			dev_err(dev, "%s: iova=0x%x, pgd=0x%x, pgprot=0x%x\n",
+				__func__, iova, imu_pgd_val(*pgd), pgprot);
+			return -1;
+		}
+		pte_new = 0;
+	}
+
+	pgprot = __m4u_get_pte_attr(prot, largepage, imu4gmode);
+	pte = imu_pte_offset_map(pgd, iova);
+
+	dev_dbg(dev, "%s:iova:0x%x,pte:0x%p(0x%p+0x%x),pa:%pa,value:0x%x-%s\n",
+		__func__, iova, &imu_pte_val(*pte), imu_pte_map(pgd),
+		imu_pte_index(iova), &pa, padscpt | pgprot,
+		largepage ? "large page" : "small page");
+
+	for (i = 0; i < ptenum; i++) {
+		if (imu_pte_val(pte[i])) {
+			dev_err(dev, "%s: pte=0x%x, i=%d\n", __func__,
+				imu_pte_val(pte[i]), i);
+			goto err_out;
+		}
+		imu_pte_val(pte[i]) = padscpt | pgprot;
+	}
+
+	m4u_pgtable_flush(pte, pte + ptenum);
+
+	return 0;
+
+ err_out:
+	for (i--; i >= 0; i--)
+		imu_pte_val(pte[i]) = 0;
+	return -EEXIST;
+}
+
+static int m4u_map_section(struct mtk_iommu_domain *m4u_domain,
+			   unsigned int iova, phys_addr_t pa,
+			   unsigned int prot, bool supersection)
+{
+	int i;
+	struct imu_pgd_t *pgd;
+	unsigned int pgprot;
+	unsigned int padscpt;
+	struct device *dev = m4u_domain->piommuinfo->dev;
+	unsigned int mask = supersection ?
+			F_PGD_PA_SUPERSECTION_MSK : F_PGD_PA_SECTION_MSK;
+	unsigned int pgdnum = supersection ? 16 : 1;
+	bool imu4gmode = (pa > 0xffffffffL) ? true : false;
+
+	if ((iova & (~mask)) != ((unsigned int)pa & (~mask))) {
+		dev_err(dev, "error to mk_pte: iova=0x%x, pa=0x%pa,type=%s\n",
+			iova, &pa, supersection ? "supersection" : "section");
+		return -EINVAL;
+	}
+
+	iova &= mask;
+	padscpt = (unsigned int)pa & mask;
+
+	pgprot = __m4u_get_pgd_attr(prot, supersection, imu4gmode);
+	pgd = imu_pgd_offset(m4u_domain, iova);
+
+	dev_dbg(dev, "%s:iova:0x%x,pgd:0x%p(0x%p+0x%x),pa:%pa,value:0x%x-%s\n",
+		__func__, iova, pgd, (m4u_domain)->pgd, imu_pgd_index(iova),
+		&pa, padscpt | pgprot,
+		supersection ? "supersection" : "section");
+
+	for (i = 0; i < pgdnum; i++) {
+		if (unlikely(imu_pgd_val(*pgd))) {
+			dev_err(dev, "%s:iova=0x%x, pgd=0x%x, i=%d\n", __func__,
+				iova, imu_pgd_val(*pgd), i);
+			goto err_out;
+		}
+		m4u_set_pgd_val(pgd, padscpt | pgprot);
+		pgd++;
+	}
+	return 0;
+
+ err_out:
+	for (pgd--; i > 0; i--) {
+		m4u_set_pgd_val(pgd, 0);
+		pgd--;
+	}
+	return -EEXIST;
+}
+
+int m4u_map(struct mtk_iommu_domain *m4u_domain, unsigned int iova,
+	    phys_addr_t paddr, unsigned int size, unsigned int prot)
+{
+	if (size == SZ_4K) {/* most case */
+		return m4u_map_page(m4u_domain, iova, paddr, prot, false);
+	} else if (size == SZ_64K) {
+		return m4u_map_page(m4u_domain, iova, paddr, prot, true);
+	} else if (size == SZ_1M) {
+		return m4u_map_section(m4u_domain, iova, paddr, prot, false);
+	} else if (size == SZ_16M) {
+		return m4u_map_section(m4u_domain, iova, paddr, prot, true);
+	} else {
+		return -EINVAL;
+	}
+}
+
+static int m4u_check_free_pte(struct mtk_iommu_domain *domain,
+			      struct imu_pgd_t *pgd)
+{
+	struct imu_pte_t *pte;
+	int i;
+
+	pte = imu_pte_map(pgd);
+	for (i = 0; i < IMU_PTRS_PER_PTE; i++, pte++) {
+		if (imu_pte_val(*pte) != 0)
+			return 1;
+	}
+
+	m4u_free_pte(domain, pgd);
+	return 0;
+}
+
+int m4u_unmap(struct mtk_iommu_domain *domain, unsigned int iova,
+	      unsigned int size)
+{
+	struct imu_pgd_t *pgd;
+	int i, ret;
+	unsigned long end_plus_1 = (unsigned long)iova + size;
+
+	do {
+		pgd = imu_pgd_offset(domain, iova);
+
+		if (F_PGD_TYPE_IS_PAGE(*pgd)) {
+			struct imu_pte_t *pte;
+			unsigned int pte_offset;
+			unsigned int num_to_clean;
+
+			pte_offset = imu_pte_index(iova);
+			num_to_clean =
+			    min((unsigned int)((end_plus_1 - iova) / PAGE_SIZE),
+				(unsigned int)(IMU_PTRS_PER_PTE - pte_offset));
+
+			pte = imu_pte_offset_map(pgd, iova);
+
+			memset(pte, 0, num_to_clean << 2);
+
+			ret = m4u_check_free_pte(domain, pgd);
+			if (ret == 1)/* pte is not freed, need to flush pte */
+				m4u_pgtable_flush(pte, pte + num_to_clean);
+
+			iova += num_to_clean << PAGE_SHIFT;
+		} else if (F_PGD_TYPE_IS_SECTION(*pgd)) {
+			m4u_set_pgd_val(pgd, 0);
+			iova += MMU_SECTION_SIZE;
+		} else if (F_PGD_TYPE_IS_SUPERSECTION(*pgd)) {
+			struct imu_pgd_t *start = imu_supersection_start(pgd);
+
+			if (unlikely(start != pgd))
+				dev_err(domain->piommuinfo->dev,
+					"%s:supper not align,iova=0x%x,pgd=0x%x\n",
+					__func__, iova, imu_pgd_val(*pgd));
+
+			for (i = 0; i < 16; i++)
+				m4u_set_pgd_val((start+i), 0);
+
+			iova = (iova + MMU_SUPERSECTION_SIZE) &
+				(~(MMU_SUPERSECTION_SIZE - 1));
+		} else {
+			iova += MMU_SECTION_SIZE;
+		}
+	} while (iova < end_plus_1 && iova);
+
+	return 0;
+}
+
+int m4u_get_pte_info(const struct mtk_iommu_domain *domain, unsigned int iova,
+		     struct m4u_pte_info_t *pte_info)
+{
+	struct imu_pgd_t *pgd;
+	struct imu_pte_t *pte;
+	unsigned int pa = 0;
+	unsigned int size;
+	int valid = 1;
+
+	pgd = imu_pgd_offset(domain, iova);
+
+	if (F_PGD_TYPE_IS_PAGE(*pgd)) {
+		pte = imu_pte_offset_map(pgd, iova);
+		if (F_PTE_TYPE_GET(imu_pte_val(*pte)) == F_PTE_TYPE_LARGE) {
+			pa = imu_pte_val(*pte) & F_PTE_PA_LARGE_MSK;
+			pa |= iova & (~F_PTE_PA_LARGE_MSK);
+			size = MMU_LARGE_PAGE_SIZE;
+		} else if (F_PTE_TYPE_GET(imu_pte_val(*pte))
+			   == F_PTE_TYPE_SMALL) {
+			pa = imu_pte_val(*pte) & F_PTE_PA_SMALL_MSK;
+			pa |= iova & (~F_PTE_PA_SMALL_MSK);
+			size = MMU_SMALL_PAGE_SIZE;
+		} else {
+			valid = 0;
+			size = MMU_SMALL_PAGE_SIZE;
+		}
+	} else {
+		pte = NULL;
+		if (F_PGD_TYPE_IS_SECTION(*pgd)) {
+			pa = imu_pgd_val(*pgd) & F_PGD_PA_SECTION_MSK;
+			pa |= iova & (~F_PGD_PA_SECTION_MSK);
+			size = MMU_SECTION_SIZE;
+		} else if (F_PGD_TYPE_IS_SUPERSECTION(*pgd)) {
+			pa = imu_pgd_val(*pgd) & F_PGD_PA_SUPERSECTION_MSK;
+			pa |= iova & (~F_PGD_PA_SUPERSECTION_MSK);
+			size = MMU_SUPERSECTION_SIZE;
+		} else {
+			valid = 0;
+			size = MMU_SECTION_SIZE;
+		}
+	}
+
+	pte_info->pgd = pgd;
+	pte_info->pte = pte;
+	pte_info->iova = iova;
+	pte_info->pa = pa;
+	pte_info->size = size;
+	pte_info->valid = valid;
+	return 0;
+}
+
diff --git a/drivers/iommu/mtk_iommu_pagetable.h b/drivers/iommu/mtk_iommu_pagetable.h
new file mode 100644
index 0000000..ebdfc6c
--- /dev/null
+++ b/drivers/iommu/mtk_iommu_pagetable.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2014-2015 MediaTek Inc.
+ * Author: Yong Wu <yong.wu at mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef MTK_IOMMU_PAGETABLE_H
+#define MTK_IOMMU_PAGETABLE_H
+
+#define MMU_SMALL_PAGE_SIZE     (SZ_4K)
+#define MMU_LARGE_PAGE_SIZE     (SZ_64K)
+#define MMU_SECTION_SIZE        (SZ_1M)
+#define MMU_SUPERSECTION_SIZE   (SZ_16M)
+
+#define IMU_PGDIR_SHIFT   20
+#define IMU_PAGE_SHIFT    12
+#define IMU_PTRS_PER_PGD  4096
+#define IMU_PTRS_PER_PTE  256
+#define IMU_BYTES_PER_PTE (IMU_PTRS_PER_PTE*sizeof(unsigned int))
+
+struct imu_pte_t {
+	unsigned int imu_pte;
+};
+
+struct imu_pgd_t {
+	unsigned int imu_pgd;
+};
+
+#define imu_pte_val(x)      ((x).imu_pte)
+#define imu_pgd_val(x)      ((x).imu_pgd)
+
+struct m4u_pte_info_t {
+	struct imu_pgd_t *pgd;
+	struct imu_pte_t *pte;
+	unsigned int iova;
+	phys_addr_t pa;
+	unsigned int size;
+	int valid;
+};
+
+#endif
+
-- 
1.8.1.1.dirty




More information about the linux-arm-kernel mailing list