[RFC 01/11] soc: mediatek: MediaTek Command Queue (CMDQ) driver
Ulrich Hecht
ulrich.hecht+renesas at gmail.com
Fri Sep 29 06:09:03 PDT 2017
Ported from chromeos-3.18 kernel.
Signed-off-by: Ulrich Hecht <ulrich.hecht+renesas at gmail.com>
---
drivers/soc/mediatek/Kconfig | 10 +
drivers/soc/mediatek/Makefile | 1 +
drivers/soc/mediatek/mtk-cmdq.c | 2814 +++++++++++++++++++++++++++++++++++++++
include/soc/mediatek/cmdq.h | 211 +++
4 files changed, 3036 insertions(+)
create mode 100644 drivers/soc/mediatek/mtk-cmdq.c
create mode 100644 include/soc/mediatek/cmdq.h
diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
index 609bb34..ef271e0 100644
--- a/drivers/soc/mediatek/Kconfig
+++ b/drivers/soc/mediatek/Kconfig
@@ -1,6 +1,16 @@
#
# MediaTek SoC drivers
#
+config MTK_CMDQ
+ bool "MediaTek CMDQ Support"
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ select MTK_INFRACFG
+ help
+ Say yes here to add support for the MediaTek Command Queue (CMDQ)
+ driver. The CMDQ is used to help read/write registers with critical
+ time limitation, such as updating display configuration during the
+ vblank.
+
config MTK_INFRACFG
bool "MediaTek INFRACFG Support"
depends on ARCH_MEDIATEK || COMPILE_TEST
diff --git a/drivers/soc/mediatek/Makefile b/drivers/soc/mediatek/Makefile
index 12998b0..f7397ef 100644
--- a/drivers/soc/mediatek/Makefile
+++ b/drivers/soc/mediatek/Makefile
@@ -1,3 +1,4 @@
+obj-$(CONFIG_MTK_CMDQ) += mtk-cmdq.o
obj-$(CONFIG_MTK_INFRACFG) += mtk-infracfg.o
obj-$(CONFIG_MTK_PMIC_WRAP) += mtk-pmic-wrap.o
obj-$(CONFIG_MTK_SCPSYS) += mtk-scpsys.o
diff --git a/drivers/soc/mediatek/mtk-cmdq.c b/drivers/soc/mediatek/mtk-cmdq.c
new file mode 100644
index 0000000..a8bfb5c
--- /dev/null
+++ b/drivers/soc/mediatek/mtk-cmdq.c
@@ -0,0 +1,2814 @@
+/*
+ * Copyright (c) 2015 MediaTek Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/ktime.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <soc/mediatek/cmdq.h>
+
+/*
+ * Please calculate this value for each platform.
+ * task number = vblank time / ((task cmds * cmd ticks) / GCE freq)
+ */
+#define CMDQ_MAX_TASK_IN_THREAD 70
+
+#define CMDQ_INITIAL_CMD_BLOCK_SIZE PAGE_SIZE
+#define CMDQ_CMD_BUF_POOL_BUF_SIZE PAGE_SIZE
+#define CMDQ_CMD_BUF_POOL_BUF_NUM 140 /* 2 * 70 = 140 */
+#define CMDQ_INST_SIZE 8 /* instruction is 64-bit */
+
+/*
+ * cmdq_thread cookie value is from 0 to CMDQ_MAX_COOKIE_VALUE.
+ * And, this value also be used as MASK.
+ */
+#define CMDQ_MAX_COOKIE_VALUE 0xffff
+#define CMDQ_COOKIE_MASK CMDQ_MAX_COOKIE_VALUE
+
+#define CMDQ_DEFAULT_TIMEOUT_MS 1000
+#define CMDQ_ACQUIRE_THREAD_TIMEOUT_MS 5000
+#define CMDQ_PREALARM_TIMEOUT_NS 200000000
+
+#define CMDQ_INVALID_THREAD -1
+
+#define CMDQ_DRIVER_DEVICE_NAME "mtk_cmdq"
+
+#define CMDQ_CLK_NAME "gce"
+
+#define CMDQ_CURR_IRQ_STATUS_OFFSET 0x010
+#define CMDQ_CURR_LOADED_THR_OFFSET 0x018
+#define CMDQ_THR_SLOT_CYCLES_OFFSET 0x030
+#define CMDQ_THR_EXEC_CYCLES_OFFSET 0x034
+#define CMDQ_THR_TIMEOUT_TIMER_OFFSET 0x038
+#define CMDQ_BUS_CONTROL_TYPE_OFFSET 0x040
+
+#define CMDQ_SYNC_TOKEN_ID_OFFSET 0x060
+#define CMDQ_SYNC_TOKEN_VAL_OFFSET 0x064
+#define CMDQ_SYNC_TOKEN_UPD_OFFSET 0x068
+
+#define CMDQ_GPR_SHIFT 0x004
+#define CMDQ_GPR_OFFSET 0x080
+
+#define CMDQ_THR_SHIFT 0x080
+#define CMDQ_THR_WARM_RESET_OFFSET 0x100
+#define CMDQ_THR_ENABLE_TASK_OFFSET 0x104
+#define CMDQ_THR_SUSPEND_TASK_OFFSET 0x108
+#define CMDQ_THR_CURR_STATUS_OFFSET 0x10c
+#define CMDQ_THR_IRQ_STATUS_OFFSET 0x110
+#define CMDQ_THR_IRQ_ENABLE_OFFSET 0x114
+#define CMDQ_THR_CURR_ADDR_OFFSET 0x120
+#define CMDQ_THR_END_ADDR_OFFSET 0x124
+#define CMDQ_THR_EXEC_CNT_OFFSET 0x128
+#define CMDQ_THR_WAIT_TOKEN_OFFSET 0x130
+#define CMDQ_THR_CFG_OFFSET 0x140
+#define CMDQ_THR_INST_CYCLES_OFFSET 0x150
+#define CMDQ_THR_INST_THRESX_OFFSET 0x154
+#define CMDQ_THR_STATUS_OFFSET 0x18c
+
+#define CMDQ_SYNC_TOKEN_SET BIT(16)
+#define CMDQ_IRQ_MASK 0xffff
+
+#define CMDQ_THR_ENABLED 0x1
+#define CMDQ_THR_DISABLED 0x0
+#define CMDQ_THR_SUSPEND 0x1
+#define CMDQ_THR_RESUME 0x0
+#define CMDQ_THR_STATUS_SUSPENDED BIT(1)
+#define CMDQ_THR_WARM_RESET BIT(0)
+#define CMDQ_THR_SLOT_CYCLES 0x3200
+#define CMDQ_THR_NO_TIMEOUT 0x0
+#define CMDQ_THR_PRIORITY 3
+#define CMDQ_THR_IRQ_DONE 0x1
+#define CMDQ_THR_IRQ_ERROR 0x12
+#define CMDQ_THR_IRQ_EN 0x13 /* done + error */
+#define CMDQ_THR_IRQ_MASK 0x13
+#define CMDQ_THR_EXECUTING BIT(31)
+#define CMDQ_THR_IS_WAITING BIT(31)
+
+#define CMDQ_ARG_A_MASK 0xffffff
+#define CMDQ_ARG_A_WRITE_MASK 0xffff
+#define CMDQ_ARG_A_SUBSYS_MASK 0x1f0000
+#define CMDQ_SUBSYS_MASK 0x1f
+
+#define CMDQ_OP_CODE_SHIFT 24
+#define CMDQ_SUBSYS_SHIFT 16
+
+#define CMDQ_JUMP_BY_OFFSET 0x10000000
+#define CMDQ_JUMP_BY_PA 0x10000001
+#define CMDQ_JUMP_PASS CMDQ_INST_SIZE
+
+#define CMDQ_WFE_UPDATE BIT(31)
+#define CMDQ_WFE_WAIT BIT(15)
+#define CMDQ_WFE_WAIT_VALUE 0x1
+
+#define CMDQ_MARK_NON_SUSPENDABLE BIT(21) /* 53 - 32 = 21 */
+#define CMDQ_MARK_NOT_ADD_COUNTER BIT(16) /* 48 - 32 = 16 */
+#define CMDQ_MARK_PREFETCH_MARKER BIT(20)
+#define CMDQ_MARK_PREFETCH_MARKER_EN BIT(17)
+#define CMDQ_MARK_PREFETCH_EN BIT(16)
+
+#define CMDQ_EOC_IRQ_EN BIT(0)
+
+#define CMDQ_ENABLE_MASK BIT(0)
+
+#define CMDQ_OP_CODE_MASK 0xff000000
+
+enum cmdq_thread_index {
+ CMDQ_THR_DISP_DSI0 = 0, /* main: dsi0 */
+ CMDQ_THR_DISP_DPI0, /* sub: dpi0 */
+ CMDQ_MAX_THREAD_COUNT, /* max */
+};
+
+struct cmdq_command {
+ struct cmdq *cqctx;
+ /* bit flag of used engines */
+ u64 engine_flag;
+ /*
+ * pointer of instruction buffer
+ * This must point to an 64-bit aligned u32 array
+ */
+ u32 *va_base;
+ /* size of instruction buffer, in bytes. */
+ size_t block_size;
+};
+
+enum cmdq_code {
+ /* These are actual HW op code. */
+ CMDQ_CODE_MOVE = 0x02,
+ CMDQ_CODE_WRITE = 0x04,
+ CMDQ_CODE_JUMP = 0x10,
+ CMDQ_CODE_WFE = 0x20, /* wait for event (and clear) */
+ CMDQ_CODE_CLEAR_EVENT = 0x21, /* clear event */
+ CMDQ_CODE_EOC = 0x40, /* end of command */
+};
+
+enum cmdq_task_state {
+ TASK_STATE_IDLE, /* free task */
+ TASK_STATE_BUSY, /* task running on a thread */
+ TASK_STATE_KILLED, /* task process being killed */
+ TASK_STATE_ERROR, /* task execution error */
+ TASK_STATE_DONE, /* task finished */
+ TASK_STATE_WAITING, /* allocated but waiting for available thread */
+};
+
+struct cmdq_cmd_buf {
+ atomic_t used;
+ void *va;
+ dma_addr_t pa;
+};
+
+struct cmdq_task_cb {
+ /* called by isr */
+ cmdq_async_flush_cb isr_cb;
+ void *isr_data;
+ /* called by releasing task */
+ cmdq_async_flush_cb done_cb;
+ void *done_data;
+};
+
+struct cmdq_task {
+ struct cmdq *cqctx;
+ struct list_head list_entry;
+
+ /* state for task life cycle */
+ enum cmdq_task_state task_state;
+ /* virtual address of command buffer */
+ u32 *va_base;
+ /* physical address of command buffer */
+ dma_addr_t mva_base;
+ /* size of allocated command buffer */
+ size_t buf_size;
+ /* It points to a cmdq_cmd_buf if this task use command buffer pool. */
+ struct cmdq_cmd_buf *cmd_buf;
+
+ u64 engine_flag;
+ size_t command_size;
+ u32 num_cmd; /* 2 * number of commands */
+ int reorder;
+ /* HW thread ID; CMDQ_INVALID_THREAD if not running */
+ int thread;
+ /* flag of IRQ received */
+ int irq_flag;
+ /* callback functions */
+ struct cmdq_task_cb cb;
+ /* work item when auto release is used */
+ struct work_struct auto_release_work;
+
+ ktime_t submit; /* submit time */
+
+ pid_t caller_pid;
+ char caller_name[TASK_COMM_LEN];
+};
+
+struct cmdq_thread {
+ u32 task_count;
+ u32 wait_cookie;
+ u32 next_cookie;
+ struct cmdq_task *cur_task[CMDQ_MAX_TASK_IN_THREAD];
+};
+
+struct cmdq {
+ struct device *dev;
+
+ void __iomem *base;
+ u32 irq;
+
+ /*
+ * task information
+ * task_cache: struct cmdq_task object cache
+ * task_free_list: unused free tasks
+ * task_active_list: active tasks
+ * task_consume_wait_queue_item: task consumption work item
+ * task_auto_release_wq: auto-release workqueue
+ * task_consume_wq: task consumption workqueue (for queued tasks)
+ */
+ struct kmem_cache *task_cache;
+ struct list_head task_free_list;
+ struct list_head task_active_list;
+ struct list_head task_wait_list;
+ struct work_struct task_consume_wait_queue_item;
+ struct workqueue_struct *task_auto_release_wq;
+ struct workqueue_struct *task_consume_wq;
+
+ struct cmdq_thread thread[CMDQ_MAX_THREAD_COUNT];
+
+ /* mutex, spinlock, flag */
+ struct mutex task_mutex; /* for task list */
+ struct mutex clock_mutex; /* for clock operation */
+ spinlock_t thread_lock; /* for cmdq hardware thread */
+ int thread_usage;
+ spinlock_t exec_lock; /* for exec task */
+
+ /* command buffer pool */
+ struct cmdq_cmd_buf cmd_buf_pool[CMDQ_CMD_BUF_POOL_BUF_NUM];
+
+ /*
+ * notification
+ * wait_queue: for task done
+ * thread_dispatch_queue: for thread acquiring
+ */
+ wait_queue_head_t wait_queue[CMDQ_MAX_THREAD_COUNT];
+ wait_queue_head_t thread_dispatch_queue;
+
+ /* ccf */
+ struct clk *clock;
+};
+
+struct cmdq_event_item {
+ enum cmdq_event event;
+ const char *name;
+};
+
+struct cmdq_subsys {
+ u32 base_addr;
+ int id;
+ const char *name;
+};
+
+static const struct cmdq_event_item cmdq_events[] = {
+ /* Display start of frame(SOF) events */
+ {CMDQ_EVENT_DISP_OVL0_SOF, "CMDQ_EVENT_DISP_OVL0_SOF"},
+ {CMDQ_EVENT_DISP_OVL1_SOF, "CMDQ_EVENT_DISP_OVL1_SOF"},
+ {CMDQ_EVENT_DISP_RDMA0_SOF, "CMDQ_EVENT_DISP_RDMA0_SOF"},
+ {CMDQ_EVENT_DISP_RDMA1_SOF, "CMDQ_EVENT_DISP_RDMA1_SOF"},
+ {CMDQ_EVENT_DISP_RDMA2_SOF, "CMDQ_EVENT_DISP_RDMA2_SOF"},
+ {CMDQ_EVENT_DISP_WDMA0_SOF, "CMDQ_EVENT_DISP_WDMA0_SOF"},
+ {CMDQ_EVENT_DISP_WDMA1_SOF, "CMDQ_EVENT_DISP_WDMA1_SOF"},
+ /* Display end of frame(EOF) events */
+ {CMDQ_EVENT_DISP_OVL0_EOF, "CMDQ_EVENT_DISP_OVL0_EOF"},
+ {CMDQ_EVENT_DISP_OVL1_EOF, "CMDQ_EVENT_DISP_OVL1_EOF"},
+ {CMDQ_EVENT_DISP_RDMA0_EOF, "CMDQ_EVENT_DISP_RDMA0_EOF"},
+ {CMDQ_EVENT_DISP_RDMA1_EOF, "CMDQ_EVENT_DISP_RDMA1_EOF"},
+ {CMDQ_EVENT_DISP_RDMA2_EOF, "CMDQ_EVENT_DISP_RDMA2_EOF"},
+ {CMDQ_EVENT_DISP_WDMA0_EOF, "CMDQ_EVENT_DISP_WDMA0_EOF"},
+ {CMDQ_EVENT_DISP_WDMA1_EOF, "CMDQ_EVENT_DISP_WDMA1_EOF"},
+ /* Mutex end of frame(EOF) events */
+ {CMDQ_EVENT_MUTEX0_STREAM_EOF, "CMDQ_EVENT_MUTEX0_STREAM_EOF"},
+ {CMDQ_EVENT_MUTEX1_STREAM_EOF, "CMDQ_EVENT_MUTEX1_STREAM_EOF"},
+ {CMDQ_EVENT_MUTEX2_STREAM_EOF, "CMDQ_EVENT_MUTEX2_STREAM_EOF"},
+ {CMDQ_EVENT_MUTEX3_STREAM_EOF, "CMDQ_EVENT_MUTEX3_STREAM_EOF"},
+ {CMDQ_EVENT_MUTEX4_STREAM_EOF, "CMDQ_EVENT_MUTEX4_STREAM_EOF"},
+ /* Display underrun events */
+ {CMDQ_EVENT_DISP_RDMA0_UNDERRUN, "CMDQ_EVENT_DISP_RDMA0_UNDERRUN"},
+ {CMDQ_EVENT_DISP_RDMA1_UNDERRUN, "CMDQ_EVENT_DISP_RDMA1_UNDERRUN"},
+ {CMDQ_EVENT_DISP_RDMA2_UNDERRUN, "CMDQ_EVENT_DISP_RDMA2_UNDERRUN"},
+ /* Keep this at the end of HW events */
+ {CMDQ_MAX_HW_EVENT_COUNT, "CMDQ_MAX_HW_EVENT_COUNT"},
+ /* This is max event and also can be used as mask. */
+ {CMDQ_SYNC_TOKEN_MAX, "CMDQ_SYNC_TOKEN_MAX"},
+ /* Invalid event */
+ {CMDQ_SYNC_TOKEN_INVALID, "CMDQ_SYNC_TOKEN_INVALID"},
+};
+
+static const struct cmdq_subsys g_subsys[] = {
+ {0x1400, 1, "MMSYS"},
+ {0x1401, 2, "DISP"},
+ {0x1402, 3, "DISP"},
+};
+
+static const char *cmdq_event_get_name(enum cmdq_event event)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(cmdq_events); i++)
+ if (cmdq_events[i].event == event)
+ return cmdq_events[i].name;
+
+ return "CMDQ_EVENT_UNKNOWN";
+}
+
+static void cmdq_event_reset(struct cmdq *cqctx)
+{
+ int i;
+
+ /* set all defined HW events to 0 */
+ for (i = 0; i < ARRAY_SIZE(cmdq_events); i++) {
+ if (cmdq_events[i].event >= CMDQ_MAX_HW_EVENT_COUNT)
+ break;
+ writel(cmdq_events[i].event,
+ cqctx->base + CMDQ_SYNC_TOKEN_UPD_OFFSET);
+ }
+}
+
+static int cmdq_subsys_base_addr_to_id(u32 base_addr)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
+ if (g_subsys[i].base_addr == base_addr)
+ return g_subsys[i].id;
+ }
+
+ return -EFAULT;
+}
+
+static u32 cmdq_subsys_id_to_base_addr(int id)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
+ if (g_subsys[i].id == id)
+ return g_subsys[i].base_addr;
+ }
+
+ return 0;
+}
+
+static const char *cmdq_subsys_base_addr_to_name(u32 base_addr)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(g_subsys); i++)
+ if (g_subsys[i].base_addr == base_addr)
+ return g_subsys[i].name;
+
+ return NULL;
+}
+
+static int cmdq_eng_get_thread(u64 flag)
+{
+ if (flag & BIT_ULL(CMDQ_ENG_DISP_DSI0))
+ return CMDQ_THR_DISP_DSI0;
+ else /* CMDQ_ENG_DISP_DPI0 */
+ return CMDQ_THR_DISP_DPI0;
+}
+
+static const char *cmdq_event_get_module(enum cmdq_event event)
+{
+ const char *module;
+
+ switch (event) {
+ case CMDQ_EVENT_DISP_RDMA0_SOF:
+ case CMDQ_EVENT_DISP_RDMA1_SOF:
+ case CMDQ_EVENT_DISP_RDMA2_SOF:
+ case CMDQ_EVENT_DISP_RDMA0_EOF:
+ case CMDQ_EVENT_DISP_RDMA1_EOF:
+ case CMDQ_EVENT_DISP_RDMA2_EOF:
+ case CMDQ_EVENT_DISP_RDMA0_UNDERRUN:
+ case CMDQ_EVENT_DISP_RDMA1_UNDERRUN:
+ case CMDQ_EVENT_DISP_RDMA2_UNDERRUN:
+ module = "DISP_RDMA";
+ break;
+ case CMDQ_EVENT_DISP_WDMA0_SOF:
+ case CMDQ_EVENT_DISP_WDMA1_SOF:
+ case CMDQ_EVENT_DISP_WDMA0_EOF:
+ case CMDQ_EVENT_DISP_WDMA1_EOF:
+ module = "DISP_WDMA";
+ break;
+ case CMDQ_EVENT_DISP_OVL0_SOF:
+ case CMDQ_EVENT_DISP_OVL1_SOF:
+ case CMDQ_EVENT_DISP_OVL0_EOF:
+ case CMDQ_EVENT_DISP_OVL1_EOF:
+ module = "DISP_OVL";
+ break;
+ case CMDQ_EVENT_MUTEX0_STREAM_EOF ... CMDQ_EVENT_MUTEX4_STREAM_EOF:
+ module = "DISP";
+ break;
+ default:
+ module = "CMDQ";
+ break;
+ }
+
+ return module;
+}
+
+static u32 cmdq_thread_get_cookie(struct cmdq *cqctx, int tid)
+{
+ return readl(cqctx->base + CMDQ_THR_EXEC_CNT_OFFSET +
+ CMDQ_THR_SHIFT * tid) & CMDQ_COOKIE_MASK;
+}
+
+static int cmdq_cmd_buf_pool_init(struct cmdq *cqctx)
+{
+ struct device *dev = cqctx->dev;
+ int i;
+ int ret = 0;
+ struct cmdq_cmd_buf *buf;
+
+ for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
+ buf = &cqctx->cmd_buf_pool[i];
+ buf->va = dma_alloc_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE,
+ &buf->pa, GFP_KERNEL);
+ if (!buf->va) {
+ dev_err(dev, "failed to alloc cmdq_cmd_buf\n");
+ ret = -ENOMEM;
+ goto fail_alloc;
+ }
+ }
+
+ return 0;
+
+fail_alloc:
+ for (i -= 1; i >= 0 ; i--) {
+ buf = &cqctx->cmd_buf_pool[i];
+ dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
+ buf->pa);
+ }
+
+ return ret;
+}
+
+static void cmdq_cmd_buf_pool_uninit(struct cmdq *cqctx)
+{
+ struct device *dev = cqctx->dev;
+ int i;
+ struct cmdq_cmd_buf *buf;
+
+ for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
+ buf = &cqctx->cmd_buf_pool[i];
+ dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
+ buf->pa);
+ if (atomic_read(&buf->used))
+ dev_err(dev,
+ "cmdq_cmd_buf[%d] va:0x%p still in use\n",
+ i, buf->va);
+ }
+}
+
+static struct cmdq_cmd_buf *cmdq_cmd_buf_pool_get(struct cmdq *cqctx)
+{
+ int i;
+ struct cmdq_cmd_buf *buf;
+
+ for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
+ buf = &cqctx->cmd_buf_pool[i];
+ if (!atomic_cmpxchg(&buf->used, 0, 1))
+ return buf;
+ }
+
+ return NULL;
+}
+
+static void cmdq_cmd_buf_pool_put(struct cmdq_cmd_buf *buf)
+{
+ atomic_set(&buf->used, 0);
+}
+
+static int cmdq_subsys_from_phys_addr(struct cmdq *cqctx, u32 cmdq_phys_addr)
+{
+ u32 base_addr = cmdq_phys_addr >> 16;
+ int subsys = cmdq_subsys_base_addr_to_id(base_addr);
+
+ if (subsys < 0)
+ dev_err(cqctx->dev,
+ "unknown subsys: error=%d, phys=0x%08x\n",
+ subsys, cmdq_phys_addr);
+
+ return subsys;
+}
+
+/*
+ * It's a kmemcache creator for cmdq_task to initialize variables
+ * without command buffer.
+ */
+static void cmdq_task_ctor(void *param)
+{
+ struct cmdq_task *task = param;
+
+ memset(task, 0, sizeof(*task));
+ INIT_LIST_HEAD(&task->list_entry);
+ task->task_state = TASK_STATE_IDLE;
+ task->thread = CMDQ_INVALID_THREAD;
+}
+
+static void cmdq_task_free_command_buffer(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+
+ if (!task->va_base)
+ return;
+
+ if (task->cmd_buf)
+ cmdq_cmd_buf_pool_put(task->cmd_buf);
+ else
+ dma_free_coherent(dev, task->buf_size, task->va_base,
+ task->mva_base);
+
+ task->va_base = NULL;
+ task->mva_base = 0;
+ task->buf_size = 0;
+ task->command_size = 0;
+ task->num_cmd = 0;
+ task->cmd_buf = NULL;
+}
+
+/*
+ * Ensure size of command buffer in the given cmdq_task.
+ * Existing buffer data will be copied to new buffer.
+ * This buffer is guaranteed to be physically continuous.
+ * returns -ENOMEM if cannot allocate new buffer
+ */
+static int cmdq_task_realloc_command_buffer(struct cmdq_task *task, size_t size)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ void *new_buf = NULL;
+ dma_addr_t new_mva_base;
+ size_t cmd_size;
+ u32 num_cmd;
+ struct cmdq_cmd_buf *cmd_buf = NULL;
+
+ if (task->va_base && task->buf_size >= size)
+ return 0;
+
+ /* try command pool first */
+ if (size <= CMDQ_CMD_BUF_POOL_BUF_SIZE) {
+ cmd_buf = cmdq_cmd_buf_pool_get(cqctx);
+ if (cmd_buf) {
+ new_buf = cmd_buf->va;
+ new_mva_base = cmd_buf->pa;
+ memset(new_buf, 0, CMDQ_CMD_BUF_POOL_BUF_SIZE);
+ }
+ }
+
+ if (!new_buf) {
+ new_buf = dma_alloc_coherent(dev, size, &new_mva_base,
+ GFP_KERNEL);
+ if (!new_buf) {
+ dev_err(dev, "realloc cmd buffer of size %zu failed\n",
+ size);
+ return -ENOMEM;
+ }
+ }
+
+ /* copy and release old buffer */
+ if (task->va_base)
+ memcpy(new_buf, task->va_base, task->buf_size);
+
+ /*
+ * we should keep track of num_cmd and cmd_size
+ * since they are cleared in free command buffer
+ */
+ num_cmd = task->num_cmd;
+ cmd_size = task->command_size;
+ cmdq_task_free_command_buffer(task);
+
+ /* attach the new buffer */
+ task->va_base = new_buf;
+ task->mva_base = new_mva_base;
+ task->buf_size = cmd_buf ? CMDQ_CMD_BUF_POOL_BUF_SIZE : size;
+ task->num_cmd = num_cmd;
+ task->command_size = cmd_size;
+ task->cmd_buf = cmd_buf;
+
+ return 0;
+}
+
+/* allocate and initialize struct cmdq_task and its command buffer */
+static struct cmdq_task *cmdq_task_create(struct cmdq *cqctx)
+{
+ struct device *dev = cqctx->dev;
+ struct cmdq_task *task;
+ int status;
+
+ task = kmem_cache_alloc(cqctx->task_cache, GFP_KERNEL);
+ task->cqctx = cqctx;
+ status = cmdq_task_realloc_command_buffer(
+ task, CMDQ_INITIAL_CMD_BLOCK_SIZE);
+ if (status < 0) {
+ dev_err(dev, "allocate command buffer failed\n");
+ kmem_cache_free(cqctx->task_cache, task);
+ return NULL;
+ }
+ return task;
+}
+
+static int cmdq_dev_init(struct platform_device *pdev, struct cmdq *cqctx)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *node = dev->of_node;
+ struct resource *res;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ cqctx->base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(cqctx->base)) {
+ dev_err(dev, "failed to ioremap gce\n");
+ return PTR_ERR(cqctx->base);
+ }
+
+ cqctx->irq = irq_of_parse_and_map(node, 0);
+ if (!cqctx->irq) {
+ dev_err(dev, "failed to get irq\n");
+ return -EINVAL;
+ }
+
+ dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n",
+ dev, cqctx->base, cqctx->irq);
+ return 0;
+}
+
+static void cmdq_task_release_unlocked(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+
+ /* This func should be inside cqctx->task_mutex mutex */
+ lockdep_assert_held(&cqctx->task_mutex);
+
+ task->task_state = TASK_STATE_IDLE;
+ task->thread = CMDQ_INVALID_THREAD;
+
+ cmdq_task_free_command_buffer(task);
+
+ /*
+ * move from active/waiting list to free list
+ * todo: shrink free list
+ */
+ list_move_tail(&task->list_entry, &cqctx->task_free_list);
+}
+
+static void cmdq_task_release_internal(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+
+ mutex_lock(&cqctx->task_mutex);
+ cmdq_task_release_unlocked(task);
+ mutex_unlock(&cqctx->task_mutex);
+}
+
+static struct cmdq_task *cmdq_core_find_free_task(struct cmdq *cqctx)
+{
+ struct cmdq_task *task;
+
+ mutex_lock(&cqctx->task_mutex);
+
+ /*
+ * Pick from free list first;
+ * create one if there is no free entry.
+ */
+ if (list_empty(&cqctx->task_free_list)) {
+ task = cmdq_task_create(cqctx);
+ } else {
+ task = list_first_entry(&cqctx->task_free_list,
+ struct cmdq_task, list_entry);
+ /* remove from free list */
+ list_del_init(&task->list_entry);
+ }
+
+ mutex_unlock(&cqctx->task_mutex);
+
+ return task;
+}
+
+/* After dropping error task, we have to reorder remaining valid tasks. */
+static void cmdq_thread_reorder_task_array(struct cmdq_thread *thread,
+ int prev_id)
+{
+ int i, j;
+ int next_id, search_id;
+ int reorder_count = 0;
+ struct cmdq_task *task;
+
+ next_id = prev_id + 1;
+ for (i = 1; i < (CMDQ_MAX_TASK_IN_THREAD - 1); i++, next_id++) {
+ if (next_id >= CMDQ_MAX_TASK_IN_THREAD)
+ next_id = 0;
+
+ if (thread->cur_task[next_id])
+ break;
+
+ search_id = next_id + 1;
+ for (j = (i + 1); j < CMDQ_MAX_TASK_IN_THREAD;
+ j++, search_id++) {
+ if (search_id >= CMDQ_MAX_TASK_IN_THREAD)
+ search_id = 0;
+
+ if (thread->cur_task[search_id]) {
+ thread->cur_task[next_id] =
+ thread->cur_task[search_id];
+ thread->cur_task[search_id] = NULL;
+ if ((j - i) > reorder_count)
+ reorder_count = j - i;
+
+ break;
+ }
+ }
+
+ task = thread->cur_task[next_id];
+ if ((task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_OFFSET) &&
+ (task->va_base[task->num_cmd - 2] == CMDQ_JUMP_PASS)) {
+ /* We reached the last task */
+ break;
+ }
+ }
+
+ thread->next_cookie -= reorder_count;
+}
+
+static int cmdq_core_sync_command(struct cmdq_task *task,
+ struct cmdq_command *cmd_desc)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ int status;
+ size_t size;
+
+ size = task->command_size + CMDQ_INST_SIZE;
+ status = cmdq_task_realloc_command_buffer(task, size);
+ if (status < 0) {
+ dev_err(dev, "failed to realloc command buffer\n");
+ dev_err(dev, "task=0x%p, request size=%zu\n", task, size);
+ return status;
+ }
+
+ /* copy the commands to our DMA buffer */
+ memcpy(task->va_base, cmd_desc->va_base, cmd_desc->block_size);
+
+ /* re-adjust num_cmd according to command_size */
+ task->num_cmd = task->command_size / sizeof(task->va_base[0]);
+
+ return 0;
+}
+
+static struct cmdq_task *cmdq_core_acquire_task(struct cmdq_command *cmd_desc,
+ struct cmdq_task_cb *cb)
+{
+ struct cmdq *cqctx = cmd_desc->cqctx;
+ struct device *dev = cqctx->dev;
+ struct cmdq_task *task;
+
+ task = cmdq_core_find_free_task(cqctx);
+ if (!task) {
+ dev_err(dev, "can't acquire task info\n");
+ return NULL;
+ }
+
+ /* initialize field values */
+ task->engine_flag = cmd_desc->engine_flag;
+ task->task_state = TASK_STATE_WAITING;
+ task->reorder = 0;
+ task->thread = CMDQ_INVALID_THREAD;
+ task->irq_flag = 0x0;
+ if (cb)
+ task->cb = *cb;
+ else
+ memset(&task->cb, 0, sizeof(task->cb));
+ task->command_size = cmd_desc->block_size;
+
+ /* store caller info for debug */
+ if (current) {
+ task->caller_pid = current->pid;
+ memcpy(task->caller_name, current->comm, sizeof(current->comm));
+ }
+
+ if (cmdq_core_sync_command(task, cmd_desc) < 0) {
+ dev_err(dev, "fail to sync command\n");
+ cmdq_task_release_internal(task);
+ return NULL;
+ }
+
+ /* insert into waiting list to process */
+ if (task) {
+ task->submit = ktime_get();
+ mutex_lock(&cqctx->task_mutex);
+ list_add_tail(&task->list_entry, &cqctx->task_wait_list);
+ mutex_unlock(&cqctx->task_mutex);
+ }
+
+ return task;
+}
+
+static int cmdq_clk_enable(struct cmdq *cqctx)
+{
+ struct device *dev = cqctx->dev;
+ int ret = 0;
+
+ if (cqctx->thread_usage == 0) {
+ ret = clk_prepare_enable(cqctx->clock);
+ if (ret) {
+ dev_err(dev, "prepare and enable clk:%s fail\n",
+ CMDQ_CLK_NAME);
+ return ret;
+ }
+ cmdq_event_reset(cqctx);
+ }
+ cqctx->thread_usage++;
+
+ return ret;
+}
+
+static void cmdq_clk_disable(struct cmdq *cqctx)
+{
+ cqctx->thread_usage--;
+ if (cqctx->thread_usage <= 0)
+ clk_disable_unprepare(cqctx->clock);
+}
+
+static int cmdq_core_find_free_thread(struct cmdq *cqctx, int tid)
+{
+ struct cmdq_thread *thread = cqctx->thread;
+ u32 next_cookie;
+
+ /*
+ * make sure the found thread has enough space for the task;
+ * cmdq_thread->cur_task has size limitation.
+ */
+ if (thread[tid].task_count >= CMDQ_MAX_TASK_IN_THREAD) {
+ dev_warn(cqctx->dev, "thread(%d) task count = %d\n",
+ tid, thread[tid].task_count);
+ return CMDQ_INVALID_THREAD;
+ }
+
+ next_cookie = thread[tid].next_cookie % CMDQ_MAX_TASK_IN_THREAD;
+ if (thread[tid].cur_task[next_cookie]) {
+ dev_warn(cqctx->dev, "thread(%d) next cookie = %d\n",
+ tid, next_cookie);
+ return CMDQ_INVALID_THREAD;
+ }
+
+ return tid;
+}
+
+static struct cmdq_thread *cmdq_core_acquire_thread(struct cmdq *cqctx,
+ int candidate_tid)
+{
+ int tid;
+
+ tid = cmdq_core_find_free_thread(cqctx, candidate_tid);
+ if (tid != CMDQ_INVALID_THREAD) {
+ mutex_lock(&cqctx->clock_mutex);
+ cmdq_clk_enable(cqctx);
+ mutex_unlock(&cqctx->clock_mutex);
+ return &cqctx->thread[tid];
+ }
+ return NULL;
+}
+
+static void cmdq_core_release_thread(struct cmdq *cqctx, int tid)
+{
+ if (WARN_ON(tid == CMDQ_INVALID_THREAD))
+ return;
+
+ mutex_lock(&cqctx->clock_mutex);
+ cmdq_clk_disable(cqctx);
+ mutex_unlock(&cqctx->clock_mutex);
+}
+
+static void cmdq_task_remove_thread(struct cmdq_task *task)
+{
+ int tid = task->thread;
+
+ task->thread = CMDQ_INVALID_THREAD;
+ cmdq_core_release_thread(task->cqctx, tid);
+}
+
+static int cmdq_thread_suspend(struct cmdq *cqctx, int tid)
+{
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ u32 enabled;
+ u32 status;
+
+ /* write suspend bit */
+ writel(CMDQ_THR_SUSPEND,
+ gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ /* If already disabled, treat as suspended successful. */
+ enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ if (!(enabled & CMDQ_THR_ENABLED))
+ return 0;
+
+ /* poll suspended status */
+ if (readl_poll_timeout_atomic(gce_base +
+ CMDQ_THR_CURR_STATUS_OFFSET +
+ CMDQ_THR_SHIFT * tid,
+ status,
+ status & CMDQ_THR_STATUS_SUSPENDED,
+ 0, 10)) {
+ dev_err(dev, "Suspend HW thread %d failed\n", tid);
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+static void cmdq_thread_resume(struct cmdq *cqctx, int tid)
+{
+ void __iomem *gce_base = cqctx->base;
+
+ writel(CMDQ_THR_RESUME,
+ gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+}
+
+static int cmdq_thread_reset(struct cmdq *cqctx, int tid)
+{
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ u32 warm_reset;
+
+ writel(CMDQ_THR_WARM_RESET,
+ gce_base + CMDQ_THR_WARM_RESET_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ if (readl_poll_timeout_atomic(gce_base + CMDQ_THR_WARM_RESET_OFFSET +
+ CMDQ_THR_SHIFT * tid,
+ warm_reset,
+ !(warm_reset & CMDQ_THR_WARM_RESET),
+ 0, 10)) {
+ dev_err(dev, "Reset HW thread %d failed\n", tid);
+ return -EFAULT;
+ }
+
+ writel(CMDQ_THR_SLOT_CYCLES, gce_base + CMDQ_THR_SLOT_CYCLES_OFFSET);
+ return 0;
+}
+
+static int cmdq_thread_disable(struct cmdq *cqctx, int tid)
+{
+ void __iomem *gce_base = cqctx->base;
+
+ cmdq_thread_reset(cqctx, tid);
+ writel(CMDQ_THR_DISABLED,
+ gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ return 0;
+}
+
+static u32 *cmdq_task_get_pc_and_inst(const struct cmdq_task *task, int tid,
+ u32 insts[2])
+{
+ struct cmdq *cqctx;
+ void __iomem *gce_base;
+ unsigned long pc_pa;
+ u8 *pc_va;
+ u8 *cmd_end;
+
+ memset(insts, 0, sizeof(u32) * 2);
+
+ if (!task ||
+ !task->va_base ||
+ tid == CMDQ_INVALID_THREAD) {
+ pr_err("cmdq get pc failed since invalid param, task 0x%p, task->va_base:0x%p, thread:%d\n",
+ task, task->va_base, tid);
+ return NULL;
+ }
+
+ cqctx = task->cqctx;
+ gce_base = cqctx->base;
+
+ pc_pa = (unsigned long)readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ pc_va = (u8 *)task->va_base + (pc_pa - task->mva_base);
+ cmd_end = (u8 *)(task->va_base + task->num_cmd - 1);
+
+ if (((u8 *)task->va_base <= pc_va) && (pc_va <= cmd_end)) {
+ if (pc_va < cmd_end) {
+ /* get arg_a and arg_b */
+ insts[0] = readl(pc_va);
+ insts[1] = readl(pc_va + 4);
+ } else {
+ /* get arg_a and arg_b of previous cmd */
+ insts[0] = readl(pc_va - 8);
+ insts[1] = readl(pc_va - 4);
+ }
+ } else {
+ return NULL;
+ }
+
+ return (u32 *)pc_va;
+}
+
+static const char *cmdq_core_parse_module_from_subsys(u32 arg_a)
+{
+ int id = (arg_a & CMDQ_ARG_A_SUBSYS_MASK) >> CMDQ_SUBSYS_SHIFT;
+ u32 base_addr = cmdq_subsys_id_to_base_addr(id);
+ const char *module = cmdq_subsys_base_addr_to_name(base_addr);
+
+ return module ? module : "CMDQ";
+}
+
+static const char *cmdq_core_parse_op(u32 op_code)
+{
+ switch (op_code) {
+ case CMDQ_CODE_WRITE:
+ return "WRIT";
+ case CMDQ_CODE_WFE:
+ return "SYNC";
+ case CMDQ_CODE_MOVE:
+ return "MASK";
+ case CMDQ_CODE_JUMP:
+ return "JUMP";
+ case CMDQ_CODE_EOC:
+ return "MARK";
+ }
+ return NULL;
+}
+
+static void cmdq_core_parse_error(struct cmdq_task *task, int tid,
+ const char **module_name, int *flag,
+ u32 *inst_a, u32 *inst_b)
+{
+ int irq_flag = task->irq_flag;
+ u32 insts[2] = { 0 };
+ const char *module;
+
+ /*
+ * other cases, use instruction to judge
+ * because engine flag are not sufficient
+ */
+ if (cmdq_task_get_pc_and_inst(task, tid, insts)) {
+ u32 op, arg_a, arg_b;
+
+ op = insts[1] >> CMDQ_OP_CODE_SHIFT;
+ arg_a = insts[1] & CMDQ_ARG_A_MASK;
+ arg_b = insts[0];
+
+ switch (op) {
+ case CMDQ_CODE_WRITE:
+ module = cmdq_core_parse_module_from_subsys(arg_a);
+ break;
+ case CMDQ_CODE_WFE:
+ /* arg_a is the event id */
+ module = cmdq_event_get_module((enum cmdq_event)arg_a);
+ break;
+ case CMDQ_CODE_MOVE:
+ case CMDQ_CODE_JUMP:
+ case CMDQ_CODE_EOC:
+ default:
+ module = "CMDQ";
+ break;
+ }
+ } else {
+ module = "CMDQ";
+ }
+
+ /* fill output parameter */
+ *module_name = module;
+ *flag = irq_flag;
+ *inst_a = insts[1];
+ *inst_b = insts[0];
+}
+
+static void cmdq_thread_insert_task_by_cookie(struct cmdq_thread *thread,
+ struct cmdq_task *task,
+ int cookie)
+{
+ thread->wait_cookie = cookie;
+ thread->next_cookie = cookie + 1;
+ if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
+ thread->next_cookie = 0;
+
+ /* first task, so set to 1 */
+ thread->task_count = 1;
+
+ thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
+}
+
+static int cmdq_thread_remove_task_by_index(struct cmdq_thread *thread,
+ int index,
+ enum cmdq_task_state new_state)
+{
+ struct cmdq_task *task;
+ struct device *dev;
+
+ task = thread->cur_task[index];
+ if (!task) {
+ pr_err("%s: remove fail, task:%d on thread:0x%p is NULL\n",
+ __func__, index, thread);
+ return -EINVAL;
+ }
+ dev = task->cqctx->dev;
+
+ /*
+ * note timing to switch a task to done_status(_ERROR, _KILLED, _DONE)
+ * is aligned with thread's taskcount change
+ * check task status to prevent double clean-up thread's taskcount
+ */
+ if (task->task_state != TASK_STATE_BUSY) {
+ dev_err(dev, "remove task failed\n");
+ dev_err(dev, "state:%d. thread:0x%p, task:%d, new_state:%d\n",
+ task->task_state, thread, index, new_state);
+ return -EINVAL;
+ }
+
+ if (thread->task_count == 0) {
+ dev_err(dev, "no task to remove\n");
+ dev_err(dev, "thread:%d, index:%d\n", task->thread, index);
+ return -EINVAL;
+ }
+
+ task->task_state = new_state;
+ thread->cur_task[index] = NULL;
+ thread->task_count--;
+
+ return 0;
+}
+
+static int cmdq_thread_force_remove_task(struct cmdq_task *task, int tid)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ void __iomem *gce_base = cqctx->base;
+ int status;
+ int cookie;
+ struct cmdq_task *exec_task;
+
+ status = cmdq_thread_suspend(cqctx, tid);
+
+ writel(CMDQ_THR_NO_TIMEOUT,
+ gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ /* The cookie of the task currently being processed */
+ cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
+
+ exec_task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
+ if (exec_task && exec_task == task) {
+ dma_addr_t eoc_pa = task->mva_base + task->command_size - 16;
+
+ /* The task is executed now, set the PC to EOC for bypass */
+ writel(eoc_pa,
+ gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = NULL;
+ task->task_state = TASK_STATE_KILLED;
+ } else {
+ int i, j;
+
+ j = thread->task_count;
+ for (i = cookie; j > 0; j--, i++) {
+ i %= CMDQ_MAX_TASK_IN_THREAD;
+
+ exec_task = thread->cur_task[i];
+ if (!exec_task)
+ continue;
+
+ if ((exec_task->va_base[exec_task->num_cmd - 1] ==
+ CMDQ_JUMP_BY_OFFSET) &&
+ (exec_task->va_base[exec_task->num_cmd - 2] ==
+ CMDQ_JUMP_PASS)) {
+ /* reached the last task */
+ break;
+ }
+
+ if (exec_task->va_base[exec_task->num_cmd - 2] ==
+ task->mva_base) {
+ /* fake EOC command */
+ exec_task->va_base[exec_task->num_cmd - 2] =
+ CMDQ_EOC_IRQ_EN;
+ exec_task->va_base[exec_task->num_cmd - 1] =
+ CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT;
+
+ /* bypass the task */
+ exec_task->va_base[exec_task->num_cmd] =
+ task->va_base[task->num_cmd - 2];
+ exec_task->va_base[exec_task->num_cmd + 1] =
+ task->va_base[task->num_cmd - 1];
+
+ i = (i + 1) % CMDQ_MAX_TASK_IN_THREAD;
+
+ thread->cur_task[i] = NULL;
+ task->task_state = TASK_STATE_KILLED;
+ status = 0;
+ break;
+ }
+ }
+ }
+
+ return status;
+}
+
+static struct cmdq_task *cmdq_thread_search_task_by_pc(
+ const struct cmdq_thread *thread, u32 pc)
+{
+ struct cmdq_task *task;
+ int i;
+
+ for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
+ task = thread->cur_task[i];
+ if (task &&
+ pc >= task->mva_base &&
+ pc <= task->mva_base + task->command_size)
+ break;
+ }
+
+ return task;
+}
+
+/*
+ * Re-fetch thread's command buffer
+ * Use Case:
+ * If SW modifies command buffer content after SW configed commands to GCE,
+ * SW should notify GCE to re-fetch commands in order to
+ * prevent inconsistent command buffer content between DRAM and GCE's SRAM.
+ */
+static void cmdq_core_invalidate_hw_fetched_buffer(struct cmdq *cqctx,
+ int tid)
+{
+ void __iomem *pc_va;
+ u32 pc;
+
+ /*
+ * Setting HW thread PC will invoke that
+ * GCE (CMDQ HW) gives up fetched command buffer,
+ * and fetch command from DRAM to GCE's SRAM again.
+ */
+ pc_va = cqctx->base + CMDQ_THR_CURR_ADDR_OFFSET + CMDQ_THR_SHIFT * tid;
+ pc = readl(pc_va);
+ writel(pc, pc_va);
+}
+
+static int cmdq_task_insert_into_thread(struct cmdq_task *task,
+ int tid, int loop)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ struct cmdq_task *prev_task;
+ int index, prev;
+
+ /* find previous task and then link this task behind it */
+
+ index = thread->next_cookie % CMDQ_MAX_TASK_IN_THREAD;
+ prev = (index + CMDQ_MAX_TASK_IN_THREAD - 1) % CMDQ_MAX_TASK_IN_THREAD;
+
+ prev_task = thread->cur_task[prev];
+
+ /* maybe the job is killed, search a new one */
+ for (; !prev_task && loop > 1; loop--) {
+ dev_err(dev,
+ "prev_task is NULL, prev:%d, loop:%d, index:%d\n",
+ prev, loop, index);
+
+ prev--;
+ if (prev < 0)
+ prev = CMDQ_MAX_TASK_IN_THREAD - 1;
+
+ prev_task = thread->cur_task[prev];
+ }
+
+ if (!prev_task) {
+ dev_err(dev,
+ "invalid prev_task index:%d, loop:%d\n",
+ index, loop);
+ return -EFAULT;
+ }
+
+ /* insert this task */
+ thread->cur_task[index] = task;
+ /* let previous task jump to this new task */
+ prev_task->va_base[prev_task->num_cmd - 1] = CMDQ_JUMP_BY_PA;
+ prev_task->va_base[prev_task->num_cmd - 2] = task->mva_base;
+
+ /* re-fetch command buffer again. */
+ cmdq_core_invalidate_hw_fetched_buffer(cqctx, tid);
+
+ return 0;
+}
+
+static bool cmdq_command_is_wfe(u32 *cmd)
+{
+ u32 wfe_option = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT | CMDQ_WFE_WAIT_VALUE;
+ u32 wfe_op = CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT;
+
+ return (cmd[0] == wfe_option && (cmd[1] & CMDQ_OP_CODE_MASK) == wfe_op);
+}
+
+/* we assume tasks in the same display thread are waiting the same event. */
+static void cmdq_task_remove_wfe(struct cmdq_task *task)
+{
+ u32 *base = task->va_base;
+ int i;
+
+ /*
+ * Replace all WFE commands in the task command queue and
+ * replace them with JUMP_PASS.
+ */
+ for (i = 0; i < task->num_cmd; i += 2) {
+ if (cmdq_command_is_wfe(&base[i])) {
+ base[i] = CMDQ_JUMP_PASS;
+ base[i + 1] = CMDQ_JUMP_BY_OFFSET;
+ }
+ }
+}
+
+static bool cmdq_thread_is_in_wfe(struct cmdq *cqctx, int tid)
+{
+ return readl(cqctx->base + CMDQ_THR_WAIT_TOKEN_OFFSET +
+ CMDQ_THR_SHIFT * tid) & CMDQ_THR_IS_WAITING;
+}
+
+static void cmdq_thread_wait_end(struct cmdq *cqctx, int tid,
+ unsigned long end_pa)
+{
+ void __iomem *gce_base = cqctx->base;
+ unsigned long curr_pa;
+
+ if (readl_poll_timeout_atomic(
+ gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid,
+ curr_pa, curr_pa == end_pa, 1, 20)) {
+ dev_err(cqctx->dev, "GCE thread(%d) cannot run to end.\n", tid);
+ }
+}
+
+static int cmdq_task_exec_async_impl(struct cmdq_task *task, int tid)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ int status;
+ struct cmdq_thread *thread;
+ unsigned long flags;
+ int loop;
+ int minimum;
+ int cookie;
+
+ status = 0;
+ thread = &cqctx->thread[tid];
+
+ spin_lock_irqsave(&cqctx->exec_lock, flags);
+
+ /* update task's thread info */
+ task->thread = tid;
+ task->irq_flag = 0;
+ task->task_state = TASK_STATE_BUSY;
+
+ /* case 1. first task for this thread */
+ if (thread->task_count <= 0) {
+ if (cmdq_thread_reset(cqctx, tid) < 0) {
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+ return -EFAULT;
+ }
+
+ writel(CMDQ_THR_NO_TIMEOUT,
+ gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ writel(task->mva_base,
+ gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ writel(task->mva_base + task->command_size,
+ gce_base + CMDQ_THR_END_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ writel(CMDQ_THR_PRIORITY,
+ gce_base + CMDQ_THR_CFG_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ writel(CMDQ_THR_IRQ_EN,
+ gce_base + CMDQ_THR_IRQ_ENABLE_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ minimum = cmdq_thread_get_cookie(cqctx, tid);
+ cmdq_thread_insert_task_by_cookie(
+ thread, task, (minimum + 1));
+
+ /* enable HW thread */
+ writel(CMDQ_THR_ENABLED,
+ gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ } else {
+ unsigned long curr_pa, end_pa;
+
+ status = cmdq_thread_suspend(cqctx, tid);
+ if (status < 0) {
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+ return status;
+ }
+
+ writel(CMDQ_THR_NO_TIMEOUT,
+ gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ cookie = thread->next_cookie;
+
+ curr_pa = (unsigned long)readl(gce_base +
+ CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ end_pa = (unsigned long)readl(gce_base +
+ CMDQ_THR_END_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ /*
+ * case 2. If already exited WFE, wait for current task to end
+ * and then jump directly to new task.
+ */
+ if (!cmdq_thread_is_in_wfe(cqctx, tid)) {
+ cmdq_thread_resume(cqctx, tid);
+ cmdq_thread_wait_end(cqctx, tid, end_pa);
+ status = cmdq_thread_suspend(cqctx, tid);
+ if (status < 0) {
+ spin_unlock_irqrestore(&cqctx->exec_lock,
+ flags);
+ return status;
+ }
+ /* set to task directly */
+ writel(task->mva_base,
+ gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ writel(task->mva_base + task->command_size,
+ gce_base + CMDQ_THR_END_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
+ thread->task_count++;
+
+ /*
+ * case 3. If thread is still in WFE from previous task, clear
+ * WFE in new task and append to thread.
+ */
+ } else {
+ /* Current task that shuld be processed */
+ minimum = cmdq_thread_get_cookie(cqctx, tid) + 1;
+ if (minimum > CMDQ_MAX_COOKIE_VALUE)
+ minimum = 0;
+
+ /* Calculate loop count to adjust the tasks' order */
+ if (minimum <= cookie)
+ loop = cookie - minimum;
+ else
+ /* Counter wrapped */
+ loop = (CMDQ_MAX_COOKIE_VALUE - minimum + 1) +
+ cookie;
+
+ if (loop < 0) {
+ dev_err(dev, "reorder fail:\n");
+ dev_err(dev, " task count=%d\n", loop);
+ dev_err(dev, " thread=%d\n", tid);
+ dev_err(dev, " next cookie=%d\n",
+ thread->next_cookie);
+ dev_err(dev, " (HW) next cookie=%d\n",
+ minimum);
+ dev_err(dev, " task=0x%p\n", task);
+
+ spin_unlock_irqrestore(&cqctx->exec_lock,
+ flags);
+ return -EFAULT;
+ }
+
+ if (loop > CMDQ_MAX_TASK_IN_THREAD)
+ loop %= CMDQ_MAX_TASK_IN_THREAD;
+
+ status = cmdq_task_insert_into_thread(task, tid, loop);
+ if (status < 0) {
+ spin_unlock_irqrestore(
+ &cqctx->exec_lock, flags);
+ dev_err(dev,
+ "invalid task state for reorder.\n");
+ return status;
+ }
+
+ cmdq_task_remove_wfe(task);
+
+ smp_mb(); /* modify jump before enable thread */
+
+ writel(task->mva_base + task->command_size,
+ gce_base + CMDQ_THR_END_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ thread->task_count++;
+ }
+
+ thread->next_cookie += 1;
+ if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
+ thread->next_cookie = 0;
+
+ /* resume HW thread */
+ cmdq_thread_resume(cqctx, tid);
+ }
+
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+
+ return status;
+}
+
+static void cmdq_core_handle_error(struct cmdq *cqctx, int tid, int value)
+{
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ struct cmdq_thread *thread;
+ struct cmdq_task *task;
+ int cookie;
+ int count;
+ int inner;
+ int status;
+ u32 curr_pa, end_pa;
+
+ curr_pa = readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ end_pa = readl(gce_base + CMDQ_THR_END_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ dev_err(dev, "IRQ: error thread=%d, irq_flag=0x%x\n", tid, value);
+ dev_err(dev, "IRQ: Thread PC: 0x%08x, End PC:0x%08x\n",
+ curr_pa, end_pa);
+
+ thread = &cqctx->thread[tid];
+
+ cookie = cmdq_thread_get_cookie(cqctx, tid);
+
+ /*
+ * we assume error happens BEFORE EOC
+ * because it wouldn't be error if this interrupt is issue by EOC.
+ * so we should inc by 1 to locate "current" task
+ */
+ cookie++;
+
+ /* set the issued task to error state */
+ if (thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD]) {
+ task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
+ task->irq_flag = value;
+ cmdq_thread_remove_task_by_index(
+ thread, cookie % CMDQ_MAX_TASK_IN_THREAD,
+ TASK_STATE_ERROR);
+ } else {
+ dev_err(dev,
+ "IRQ: can not find task in %s, pc:0x%08x, end_pc:0x%08x\n",
+ __func__, curr_pa, end_pa);
+ if (thread->task_count <= 0) {
+ /*
+ * suspend HW thread first,
+ * so that we work in a consistent state
+ * outer function should acquire spinlock:
+ * cqctx->exec_lock
+ */
+ status = cmdq_thread_suspend(cqctx, tid);
+ if (status < 0)
+ dev_err(dev, "IRQ: suspend HW thread failed!");
+
+ cmdq_thread_disable(cqctx, tid);
+ dev_err(dev,
+ "IRQ: there is no task for thread (%d) %s\n",
+ tid, __func__);
+ }
+ }
+
+ /* set the remain tasks to done state */
+ if (thread->wait_cookie <= cookie) {
+ count = cookie - thread->wait_cookie + 1;
+ } else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
+ thread->wait_cookie) {
+ count = 0;
+ } else {
+ /* counter wrapped */
+ count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
+ (cookie + 1);
+ dev_err(dev,
+ "IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
+ thread->wait_cookie, cookie, count);
+ }
+
+ for (inner = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
+ count--, inner++) {
+ if (inner >= CMDQ_MAX_TASK_IN_THREAD)
+ inner = 0;
+
+ if (thread->cur_task[inner]) {
+ task = thread->cur_task[inner];
+ task->irq_flag = 0; /* don't know irq flag */
+ /* still call isr_cb to prevent lock */
+ if (task->cb.isr_cb)
+ task->cb.isr_cb(task->cb.isr_data);
+ cmdq_thread_remove_task_by_index(
+ thread, inner, TASK_STATE_DONE);
+ }
+ }
+
+ thread->wait_cookie = cookie + 1;
+ if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
+ thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
+ /* min cookie value is 0 */
+
+ wake_up(&cqctx->wait_queue[tid]);
+}
+
+static void cmdq_core_handle_done(struct cmdq *cqctx, int tid, int value)
+{
+ struct device *dev = cqctx->dev;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ int cookie = cmdq_thread_get_cookie(cqctx, tid);
+ int count;
+ int i;
+ struct cmdq_task *task;
+
+ if (thread->wait_cookie <= cookie) {
+ count = cookie - thread->wait_cookie + 1;
+ } else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
+ thread->wait_cookie) {
+ count = 0;
+ } else {
+ /* counter wrapped */
+ count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
+ (cookie + 1);
+ dev_err(dev,
+ "IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
+ thread->wait_cookie, cookie, count);
+ }
+
+ for (i = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
+ count--, i++) {
+ if (i >= CMDQ_MAX_TASK_IN_THREAD)
+ i = 0;
+
+ if (thread->cur_task[i]) {
+ task = thread->cur_task[i];
+ task->irq_flag = value;
+ if (task->cb.isr_cb)
+ task->cb.isr_cb(task->cb.isr_data);
+ cmdq_thread_remove_task_by_index(
+ thread, i, TASK_STATE_DONE);
+ }
+ }
+
+ thread->wait_cookie = cookie + 1;
+ if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
+ thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
+ /* min cookie value is 0 */
+
+ wake_up(&cqctx->wait_queue[tid]);
+}
+
+static void cmdq_core_handle_irq(struct cmdq *cqctx, int tid)
+{
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ unsigned long flags = 0L;
+ int value;
+ int enabled;
+ int cookie;
+
+ /*
+ * normal execution, marks tasks done and remove from thread
+ * also, handle "loop CB fail" case
+ */
+ spin_lock_irqsave(&cqctx->exec_lock, flags);
+
+ /*
+ * it is possible for another CPU core
+ * to run "release task" right before we acquire the spin lock
+ * and thus reset / disable this HW thread
+ * so we check both the IRQ flag and the enable bit of this thread
+ */
+ value = readl(gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ if (!(value & CMDQ_THR_IRQ_MASK)) {
+ dev_err(dev,
+ "IRQ: thread %d got interrupt but IRQ flag is 0x%08x\n",
+ tid, value);
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+ return;
+ }
+
+ enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ if (!(enabled & CMDQ_THR_ENABLED)) {
+ dev_err(dev,
+ "IRQ: thread %d got interrupt already disabled 0x%08x\n",
+ tid, enabled);
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+ return;
+ }
+
+ /* read HW cookie here for printing message */
+ cookie = cmdq_thread_get_cookie(cqctx, tid);
+
+ /*
+ * Move the reset IRQ before read HW cookie
+ * to prevent race condition and save the cost of suspend
+ */
+ writel(~value,
+ gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ if (value & CMDQ_THR_IRQ_ERROR)
+ cmdq_core_handle_error(cqctx, tid, value);
+ else if (value & CMDQ_THR_IRQ_DONE)
+ cmdq_core_handle_done(cqctx, tid, value);
+
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+}
+
+static int cmdq_task_exec_async(struct cmdq_task *task, int tid)
+{
+ struct device *dev = task->cqctx->dev;
+ int status;
+
+ status = cmdq_task_exec_async_impl(task, tid);
+ if (status >= 0)
+ return status;
+
+ if ((task->task_state == TASK_STATE_KILLED) ||
+ (task->task_state == TASK_STATE_ERROR)) {
+ dev_err(dev, "cmdq_task_exec_async_impl fail\n");
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+static void cmdq_core_consume_waiting_list(struct work_struct *work)
+{
+ struct list_head *p, *n = NULL;
+ bool thread_acquired;
+ ktime_t consume_time;
+ s64 waiting_time_ns;
+ bool need_log;
+ struct cmdq *cqctx;
+ struct device *dev;
+ u32 err_bits = 0;
+
+ cqctx = container_of(work, struct cmdq,
+ task_consume_wait_queue_item);
+ dev = cqctx->dev;
+
+ consume_time = ktime_get();
+
+ mutex_lock(&cqctx->task_mutex);
+
+ thread_acquired = false;
+
+ /* scan and remove (if executed) waiting tasks */
+ list_for_each_safe(p, n, &cqctx->task_wait_list) {
+ struct cmdq_task *task;
+ struct cmdq_thread *thread;
+ int tid;
+ int status;
+
+ task = list_entry(p, struct cmdq_task, list_entry);
+ tid = cmdq_eng_get_thread(task->engine_flag);
+
+ waiting_time_ns = ktime_to_ns(
+ ktime_sub(consume_time, task->submit));
+ need_log = waiting_time_ns >= CMDQ_PREALARM_TIMEOUT_NS;
+
+ /*
+ * Once waiting occur,
+ * skip following tasks to keep order of display tasks.
+ */
+ if (err_bits & BIT(tid))
+ continue;
+
+ /* acquire HW thread */
+ thread = cmdq_core_acquire_thread(cqctx, tid);
+ if (!thread) {
+ /* have to wait, remain in wait list */
+ dev_warn(dev, "acquire thread(%d) fail, need to wait\n",
+ tid);
+ if (need_log) /* task wait too long */
+ dev_warn(dev, "waiting:%lldns, task:0x%p\n",
+ waiting_time_ns, task);
+ err_bits |= BIT(tid);
+ continue;
+ }
+
+ /* some task is ready to run */
+ thread_acquired = true;
+
+ /*
+ * start execution
+ * remove from wait list and put into active list
+ */
+ list_move_tail(&task->list_entry,
+ &cqctx->task_active_list);
+
+ /* run task on thread */
+ status = cmdq_task_exec_async(task, tid);
+ if (status < 0) {
+ dev_err(dev, "%s fail, release task 0x%p\n",
+ __func__, task);
+ cmdq_task_remove_thread(task);
+ cmdq_task_release_unlocked(task);
+ task = NULL;
+ }
+ }
+
+ if (thread_acquired) {
+ /*
+ * notify some task's sw thread to change their waiting state.
+ * (if they have already called cmdq_task_wait_and_release())
+ */
+ wake_up_all(&cqctx->thread_dispatch_queue);
+ }
+
+ mutex_unlock(&cqctx->task_mutex);
+}
+
+static int cmdq_core_submit_task_async(struct cmdq_command *cmd_desc,
+ struct cmdq_task **task_out,
+ struct cmdq_task_cb *cb)
+{
+ struct cmdq *cqctx = cmd_desc->cqctx;
+
+ /* creates a new task and put into tail of waiting list */
+ *task_out = cmdq_core_acquire_task(cmd_desc, cb);
+
+ if (!(*task_out))
+ return -EFAULT;
+
+ /*
+ * Consume the waiting list.
+ * This may or may not execute the task, depending on available threads.
+ */
+ cmdq_core_consume_waiting_list(&cqctx->task_consume_wait_queue_item);
+
+ return 0;
+}
+
+static int cmdq_core_release_task(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+ int tid = task->thread;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ unsigned long flags;
+ int status;
+
+ if (tid != CMDQ_INVALID_THREAD && thread) {
+ /* this task is being executed (or queueed) on a hw thread */
+
+ /* get sw lock first to ensure atomic access hw */
+ spin_lock_irqsave(&cqctx->exec_lock, flags);
+ smp_mb(); /* make sure atomic access hw */
+
+ status = cmdq_thread_force_remove_task(task, tid);
+ if (thread->task_count > 0)
+ cmdq_thread_resume(cqctx, tid);
+
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+ wake_up(&cqctx->wait_queue[tid]);
+ }
+
+ cmdq_task_remove_thread(task);
+ cmdq_task_release_internal(task);
+ return 0;
+}
+
+struct cmdq_task_error_report {
+ bool throw_err;
+ const char *module;
+ u32 inst_a;
+ u32 inst_b;
+ u32 irq_flag;
+};
+
+static int cmdq_task_handle_error_result(
+ struct cmdq_task *task, int tid, int wait_q,
+ struct cmdq_task_error_report *error_report)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ void __iomem *gce_base = cqctx->base;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ int status = 0;
+ int i;
+ bool is_err = false;
+ struct cmdq_task *next_task;
+ struct cmdq_task *prev_task;
+ int cookie;
+ unsigned long thread_pc;
+
+ dev_err(dev,
+ "task(0x%p) state is not TASK_STATE_DONE, but %d.\n",
+ task, task->task_state);
+
+ /*
+ * Oops, that task is not done.
+ * We have several possible error cases:
+ * 1. task still running (hang / timeout)
+ * 2. IRQ pending (done or error/timeout IRQ)
+ * 3. task's SW thread has been signaled (e.g. SIGKILL)
+ */
+
+ /*
+ * suspend HW thread first,
+ * so that we work in a consistent state
+ */
+ status = cmdq_thread_suspend(cqctx, tid);
+ if (status < 0)
+ error_report->throw_err = true;
+
+ /* The cookie of the task currently being processed */
+ cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
+ thread_pc = (unsigned long)readl(gce_base +
+ CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ /* process any pending IRQ */
+ error_report->irq_flag = readl(
+ gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ if (error_report->irq_flag & CMDQ_THR_IRQ_ERROR)
+ cmdq_core_handle_error(cqctx, tid, error_report->irq_flag);
+ else if (error_report->irq_flag & CMDQ_THR_IRQ_DONE)
+ cmdq_core_handle_done(cqctx, tid, error_report->irq_flag);
+
+ writel(~error_report->irq_flag,
+ gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+
+ /* check if this task has finished after handling pending IRQ */
+ if (task->task_state == TASK_STATE_DONE)
+ return 0;
+
+ /* Then decide we are SW timeout or SIGNALed (not an error) */
+ if (!wait_q) {
+ /* SW timeout and no IRQ received */
+ is_err = true;
+ dev_err(dev, "SW timeout of task 0x%p on tid %d\n",
+ task, tid);
+ error_report->throw_err = true;
+ cmdq_core_parse_error(task, tid,
+ &error_report->module,
+ &error_report->irq_flag,
+ &error_report->inst_a,
+ &error_report->inst_b);
+ status = -ETIMEDOUT;
+ } else if (wait_q < 0) {
+ /*
+ * Task is killed.
+ * Not an error, but still need to remove.
+ */
+ is_err = false;
+
+ if (wait_q == -ERESTARTSYS)
+ dev_err(dev,
+ "Task 0x%p KILLED by wait_q = -ERESTARTSYS\n",
+ task);
+ else if (wait_q == -EINTR)
+ dev_err(dev,
+ "Task 0x%p KILLED by wait_q = -EINTR\n",
+ task);
+ else
+ dev_err(dev,
+ "Task 0x%p KILLED by wait_q = %d\n",
+ task, wait_q);
+
+ status = wait_q;
+ }
+
+ if (task->task_state == TASK_STATE_BUSY) {
+ /*
+ * if task_state is BUSY,
+ * this means we did not reach EOC,
+ * did not have error IRQ.
+ * - remove the task from thread.cur_task[]
+ * - and decrease thread.task_count
+ * NOTE: after this,
+ * the cur_task will not contain link to task anymore.
+ * and task should become TASK_STATE_ERROR
+ */
+
+ /* we find our place in thread->cur_task[]. */
+ for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
+ if (thread->cur_task[i] == task) {
+ /* update task_count and cur_task[] */
+ cmdq_thread_remove_task_by_index(
+ thread, i, is_err ?
+ TASK_STATE_ERROR :
+ TASK_STATE_KILLED);
+ break;
+ }
+ }
+ }
+
+ next_task = NULL;
+
+ /* find task's jump destination or no next task*/
+ if (task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_PA)
+ next_task = cmdq_thread_search_task_by_pc(
+ thread,
+ task->va_base[task->num_cmd - 2]);
+
+ /*
+ * Then, we try remove task from the chain of thread->cur_task.
+ * . if HW PC falls in task range
+ * . HW EXEC_CNT += 1
+ * . thread.wait_cookie += 1
+ * . set HW PC to next task head
+ * . if not, find previous task
+ * (whose jump address is task->mva_base)
+ * . check if HW PC points is not at the EOC/JUMP end
+ * . change jump to fake EOC(no IRQ)
+ * . insert jump to next task head and increase cmd buffer size
+ * . if there is no next task, set HW End Address
+ */
+ if (task->num_cmd && thread_pc >= task->mva_base &&
+ thread_pc <= (task->mva_base + task->command_size)) {
+ if (next_task) {
+ /* cookie already +1 */
+ writel(cookie,
+ gce_base + CMDQ_THR_EXEC_CNT_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ thread->wait_cookie = cookie + 1;
+ writel(next_task->mva_base,
+ gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
+ CMDQ_THR_SHIFT * tid);
+ }
+ } else {
+ prev_task = NULL;
+ for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
+ u32 *prev_va, *curr_va;
+ u32 prev_num, curr_num;
+
+ prev_task = thread->cur_task[i];
+ if (!prev_task)
+ continue;
+
+ prev_va = prev_task->va_base;
+ prev_num = prev_task->num_cmd;
+ if (!prev_num)
+ continue;
+
+ curr_va = task->va_base;
+ curr_num = task->num_cmd;
+
+ /* find which task JUMP into task */
+ if (prev_va[prev_num - 2] == task->mva_base &&
+ prev_va[prev_num - 1] == CMDQ_JUMP_BY_PA) {
+ /* Copy Jump instruction */
+ prev_va[prev_num - 2] =
+ curr_va[curr_num - 2];
+ prev_va[prev_num - 1] =
+ curr_va[curr_num - 1];
+
+ if (next_task)
+ cmdq_thread_reorder_task_array(
+ thread, i);
+
+ /*
+ * Give up fetched command,
+ * invoke CMDQ HW to re-fetch command.
+ */
+ cmdq_core_invalidate_hw_fetched_buffer(
+ cqctx, tid);
+
+ break;
+ }
+ }
+ }
+
+ return status;
+}
+
+static int cmdq_task_wait_result(struct cmdq_task *task, int tid, int wait_q)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct cmdq_thread *thread = &cqctx->thread[tid];
+ int status = 0;
+ unsigned long flags;
+ struct cmdq_task_error_report error_report = { 0 };
+
+ /*
+ * Note that although we disable IRQ, HW continues to execute
+ * so it's possible to have pending IRQ
+ */
+ spin_lock_irqsave(&cqctx->exec_lock, flags);
+
+ if (task->task_state != TASK_STATE_DONE)
+ status = cmdq_task_handle_error_result(
+ task, tid, wait_q, &error_report);
+
+ if (thread->task_count <= 0)
+ cmdq_thread_disable(cqctx, tid);
+ else
+ cmdq_thread_resume(cqctx, tid);
+
+ spin_unlock_irqrestore(&cqctx->exec_lock, flags);
+
+ if (error_report.throw_err) {
+ u32 op = error_report.inst_a >> CMDQ_OP_CODE_SHIFT;
+
+ switch (op) {
+ case CMDQ_CODE_WFE:
+ dev_err(cqctx->dev,
+ "%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:WAIT EVENT:%s\n",
+ error_report.module, error_report.irq_flag,
+ error_report.inst_a, error_report.inst_b,
+ cmdq_event_get_name(error_report.inst_a &
+ CMDQ_ARG_A_MASK));
+ break;
+ default:
+ dev_err(cqctx->dev,
+ "%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:%s\n",
+ error_report.module, error_report.irq_flag,
+ error_report.inst_a, error_report.inst_b,
+ cmdq_core_parse_op(op));
+ break;
+ }
+ }
+
+ return status;
+}
+
+static int cmdq_task_wait_done(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+ struct device *dev = cqctx->dev;
+ int wait_q;
+ int tid;
+ unsigned long timeout = msecs_to_jiffies(
+ CMDQ_ACQUIRE_THREAD_TIMEOUT_MS);
+
+ /*
+ * wait for acquire thread
+ * (this is done by cmdq_core_consume_waiting_list);
+ */
+ wait_q = wait_event_timeout(
+ cqctx->thread_dispatch_queue,
+ (task->thread != CMDQ_INVALID_THREAD), timeout);
+
+ if (!wait_q) {
+ mutex_lock(&cqctx->task_mutex);
+
+ /*
+ * it's possible that the task was just consumed now.
+ * so check again.
+ */
+ if (task->thread == CMDQ_INVALID_THREAD) {
+ /*
+ * Task may have released,
+ * or starved to death.
+ */
+ dev_err(dev,
+ "task(0x%p) timeout with invalid thread\n",
+ task);
+
+ /*
+ * remove from waiting list,
+ * so that it won't be consumed in the future
+ */
+ list_del_init(&task->list_entry);
+
+ mutex_unlock(&cqctx->task_mutex);
+ return -EINVAL;
+ }
+
+ /* valid thread, so we keep going */
+ mutex_unlock(&cqctx->task_mutex);
+ }
+
+ tid = task->thread;
+ if (tid < 0 || tid >= CMDQ_MAX_THREAD_COUNT) {
+ dev_err(dev, "invalid thread %d in %s\n", tid, __func__);
+ return -EINVAL;
+ }
+
+ /* start to wait */
+ wait_q = wait_event_timeout(task->cqctx->wait_queue[tid],
+ (task->task_state != TASK_STATE_BUSY &&
+ task->task_state != TASK_STATE_WAITING),
+ msecs_to_jiffies(CMDQ_DEFAULT_TIMEOUT_MS));
+ if (!wait_q)
+ dev_dbg(dev, "timeout!\n");
+
+ /* wake up and continue */
+ return cmdq_task_wait_result(task, tid, wait_q);
+}
+
+static int cmdq_task_wait_and_release(struct cmdq_task *task)
+{
+ struct cmdq *cqctx;
+ int status;
+
+ if (!task) {
+ pr_err("%s err ptr=0x%p\n", __func__, task);
+ return -EFAULT;
+ }
+
+ if (task->task_state == TASK_STATE_IDLE) {
+ pr_err("%s task=0x%p is IDLE\n", __func__, task);
+ return -EFAULT;
+ }
+
+ cqctx = task->cqctx;
+
+ /* wait for task finish */
+ status = cmdq_task_wait_done(task);
+
+ /* release */
+ cmdq_task_remove_thread(task);
+ cmdq_task_release_internal(task);
+
+ return status;
+}
+
+static void cmdq_core_auto_release_work(struct work_struct *work_item)
+{
+ struct cmdq_task *task;
+ int status;
+ struct cmdq_task_cb cb;
+
+ task = container_of(work_item, struct cmdq_task, auto_release_work);
+ cb = task->cb;
+ status = cmdq_task_wait_and_release(task);
+
+ /* isr fail, so call isr_cb here to prevent lock */
+ if (status && cb.isr_cb)
+ cb.isr_cb(cb.isr_data);
+
+ if (cb.done_cb)
+ cb.done_cb(cb.done_data);
+}
+
+static int cmdq_core_auto_release_task(struct cmdq_task *task)
+{
+ struct cmdq *cqctx = task->cqctx;
+
+ /*
+ * the work item is embeded in task already
+ * but we need to initialized it
+ */
+ INIT_WORK(&task->auto_release_work, cmdq_core_auto_release_work);
+ queue_work(cqctx->task_auto_release_wq, &task->auto_release_work);
+ return 0;
+}
+
+static int cmdq_core_submit_task(struct cmdq_command *cmd_desc)
+{
+ struct device *dev = cmd_desc->cqctx->dev;
+ int status;
+ struct cmdq_task *task;
+
+ status = cmdq_core_submit_task_async(cmd_desc, &task, NULL);
+ if (status < 0) {
+ dev_err(dev, "cmdq_core_submit_task_async failed=%d\n", status);
+ return status;
+ }
+
+ status = cmdq_task_wait_and_release(task);
+ if (status < 0)
+ dev_err(dev, "task(0x%p) wait fail\n", task);
+
+ return status;
+}
+
+static void cmdq_core_deinitialize(struct platform_device *pdev)
+{
+ struct cmdq *cqctx = platform_get_drvdata(pdev);
+ int i;
+ struct list_head *lists[] = {
+ &cqctx->task_free_list,
+ &cqctx->task_active_list,
+ &cqctx->task_wait_list
+ };
+
+ /*
+ * Directly destroy the auto release WQ
+ * since we're going to release tasks anyway.
+ */
+ destroy_workqueue(cqctx->task_auto_release_wq);
+ cqctx->task_auto_release_wq = NULL;
+
+ destroy_workqueue(cqctx->task_consume_wq);
+ cqctx->task_consume_wq = NULL;
+
+ /* release all tasks in both list */
+ for (i = 0; i < ARRAY_SIZE(lists); i++) {
+ struct cmdq_task *task, *tmp;
+
+ list_for_each_entry_safe(task, tmp, lists[i], list_entry) {
+ cmdq_task_free_command_buffer(task);
+ kmem_cache_free(cqctx->task_cache, task);
+ list_del(&task->list_entry);
+ }
+ }
+
+ kmem_cache_destroy(cqctx->task_cache);
+ cqctx->task_cache = NULL;
+
+ /* release command buffer pool */
+ cmdq_cmd_buf_pool_uninit(cqctx);
+}
+
+static irqreturn_t cmdq_irq_handler(int irq, void *dev)
+{
+ struct cmdq *cqctx = dev;
+ int i;
+ u32 irq_status;
+ bool handled = false;
+
+ irq_status = readl(cqctx->base + CMDQ_CURR_IRQ_STATUS_OFFSET);
+ irq_status &= CMDQ_IRQ_MASK;
+ for (i = 0;
+ irq_status != CMDQ_IRQ_MASK && i < CMDQ_MAX_THREAD_COUNT;
+ i++) {
+ /* STATUS bit set to 0 means IRQ asserted */
+ if (irq_status & BIT(i))
+ continue;
+
+ /*
+ * We mark irq_status to 1 to denote finished
+ * processing, and we can early-exit if no more
+ * threads being asserted.
+ */
+ irq_status |= BIT(i);
+
+ cmdq_core_handle_irq(cqctx, i);
+ handled = true;
+ }
+
+ if (!handled)
+ return IRQ_NONE;
+
+ queue_work(cqctx->task_consume_wq,
+ &cqctx->task_consume_wait_queue_item);
+ return IRQ_HANDLED;
+}
+
+static int cmdq_core_initialize(struct platform_device *pdev,
+ struct cmdq **cqctx)
+{
+ struct cmdq *lcqctx; /* local cmdq context */
+ int i;
+ int ret = 0;
+
+ lcqctx = devm_kzalloc(&pdev->dev, sizeof(*lcqctx), GFP_KERNEL);
+
+ /* save dev */
+ lcqctx->dev = &pdev->dev;
+
+ /* initial cmdq device related data */
+ ret = cmdq_dev_init(pdev, lcqctx);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to init cmdq device\n");
+ goto fail_dev;
+ }
+
+ /* initial mutex, spinlock */
+ mutex_init(&lcqctx->task_mutex);
+ mutex_init(&lcqctx->clock_mutex);
+ spin_lock_init(&lcqctx->thread_lock);
+ spin_lock_init(&lcqctx->exec_lock);
+
+ /* initial wait queue for notification */
+ for (i = 0; i < ARRAY_SIZE(lcqctx->wait_queue); i++)
+ init_waitqueue_head(&lcqctx->wait_queue[i]);
+ init_waitqueue_head(&lcqctx->thread_dispatch_queue);
+
+ /* create task pool */
+ lcqctx->task_cache = kmem_cache_create(
+ CMDQ_DRIVER_DEVICE_NAME "_task",
+ sizeof(struct cmdq_task),
+ __alignof__(struct cmdq_task),
+ SLAB_POISON | SLAB_HWCACHE_ALIGN | SLAB_RED_ZONE,
+ &cmdq_task_ctor);
+
+ /* initialize task lists */
+ INIT_LIST_HEAD(&lcqctx->task_free_list);
+ INIT_LIST_HEAD(&lcqctx->task_active_list);
+ INIT_LIST_HEAD(&lcqctx->task_wait_list);
+ INIT_WORK(&lcqctx->task_consume_wait_queue_item,
+ cmdq_core_consume_waiting_list);
+
+ /* initialize command buffer pool */
+ ret = cmdq_cmd_buf_pool_init(lcqctx);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to init command buffer pool\n");
+ goto fail_cmd_buf_pool;
+ }
+
+ lcqctx->task_auto_release_wq = alloc_ordered_workqueue(
+ "%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_auto_release");
+ lcqctx->task_consume_wq = alloc_ordered_workqueue(
+ "%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_task");
+
+ *cqctx = lcqctx;
+ return ret;
+
+fail_cmd_buf_pool:
+ destroy_workqueue(lcqctx->task_auto_release_wq);
+ destroy_workqueue(lcqctx->task_consume_wq);
+ kmem_cache_destroy(lcqctx->task_cache);
+
+fail_dev:
+ return ret;
+}
+
+static int cmdq_rec_realloc_cmd_buffer(struct cmdq_rec *handle, size_t size)
+{
+ void *new_buf;
+
+ new_buf = krealloc(handle->buf_ptr, size, GFP_KERNEL | __GFP_ZERO);
+ if (!new_buf)
+ return -ENOMEM;
+ handle->buf_ptr = new_buf;
+ handle->buf_size = size;
+ return 0;
+}
+
+static int cmdq_rec_stop_running_task(struct cmdq_rec *handle)
+{
+ int status;
+
+ status = cmdq_core_release_task(handle->running_task_ptr);
+ handle->running_task_ptr = NULL;
+ return status;
+}
+
+int cmdq_rec_create(struct device *dev, u64 engine_flag,
+ struct cmdq_rec **handle_ptr)
+{
+ struct cmdq *cqctx;
+ struct cmdq_rec *handle;
+ int ret;
+
+ cqctx = dev_get_drvdata(dev);
+ if (!cqctx) {
+ dev_err(dev, "cmdq context is NULL\n");
+ return -EINVAL;
+ }
+
+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
+ if (!handle)
+ return -ENOMEM;
+
+ handle->cqctx = dev_get_drvdata(dev);
+ handle->engine_flag = engine_flag;
+
+ ret = cmdq_rec_realloc_cmd_buffer(handle, CMDQ_INITIAL_CMD_BLOCK_SIZE);
+ if (ret) {
+ kfree(handle);
+ return ret;
+ }
+
+ *handle_ptr = handle;
+
+ return 0;
+}
+EXPORT_SYMBOL(cmdq_rec_create);
+
+static int cmdq_rec_append_command(struct cmdq_rec *handle,
+ enum cmdq_code code,
+ u32 arg_a, u32 arg_b)
+{
+ struct cmdq *cqctx;
+ struct device *dev;
+ int subsys;
+ u32 *cmd_ptr;
+ int ret;
+
+ cqctx = handle->cqctx;
+ dev = cqctx->dev;
+ cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
+
+ if (handle->finalized) {
+ dev_err(dev,
+ "already finalized record(cannot add more command)");
+ dev_err(dev, "handle=0x%p, tid=%d\n", handle, current->pid);
+ return -EBUSY;
+ }
+
+ /* check if we have sufficient buffer size */
+ if (unlikely(handle->block_size + CMDQ_INST_SIZE > handle->buf_size)) {
+ ret = cmdq_rec_realloc_cmd_buffer(handle, handle->buf_size * 2);
+ if (ret)
+ return ret;
+ }
+
+ /*
+ * we must re-calculate current PC
+ * because we may already insert MARKER inst.
+ */
+ cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
+
+ switch (code) {
+ case CMDQ_CODE_MOVE:
+ cmd_ptr[0] = arg_b;
+ cmd_ptr[1] = (CMDQ_CODE_MOVE << CMDQ_OP_CODE_SHIFT) |
+ (arg_a & CMDQ_ARG_A_MASK);
+ break;
+ case CMDQ_CODE_WRITE:
+ subsys = cmdq_subsys_from_phys_addr(cqctx, arg_a);
+ if (subsys < 0) {
+ dev_err(dev,
+ "unsupported memory base address 0x%08x\n",
+ arg_a);
+ return -EFAULT;
+ }
+
+ cmd_ptr[0] = arg_b;
+ cmd_ptr[1] = (CMDQ_CODE_WRITE << CMDQ_OP_CODE_SHIFT) |
+ (arg_a & CMDQ_ARG_A_WRITE_MASK) |
+ ((subsys & CMDQ_SUBSYS_MASK) << CMDQ_SUBSYS_SHIFT);
+ break;
+ case CMDQ_CODE_JUMP:
+ cmd_ptr[0] = arg_b;
+ cmd_ptr[1] = (CMDQ_CODE_JUMP << CMDQ_OP_CODE_SHIFT) |
+ (arg_a & CMDQ_ARG_A_MASK);
+ break;
+ case CMDQ_CODE_WFE:
+ /*
+ * bit 0-11: wait_value, 1
+ * bit 15: to_wait, true
+ * bit 16-27: update_value, 0
+ * bit 31: to_update, true
+ */
+ cmd_ptr[0] = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT |
+ CMDQ_WFE_WAIT_VALUE;
+ cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
+ break;
+ case CMDQ_CODE_CLEAR_EVENT:
+ /*
+ * bit 0-11: wait_value, 0
+ * bit 15: to_wait, false
+ * bit 16-27: update_value, 0
+ * bit 31: to_update, true
+ */
+ cmd_ptr[0] = CMDQ_WFE_UPDATE;
+ cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
+ break;
+ case CMDQ_CODE_EOC:
+ cmd_ptr[0] = arg_b;
+ cmd_ptr[1] = (CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT) |
+ (arg_a & CMDQ_ARG_A_MASK);
+ break;
+ default:
+ return -EFAULT;
+ }
+
+ handle->block_size += CMDQ_INST_SIZE;
+
+ return 0;
+}
+
+int cmdq_rec_reset(struct cmdq_rec *handle)
+{
+ if (handle->running_task_ptr)
+ cmdq_rec_stop_running_task(handle);
+
+ handle->block_size = 0;
+ handle->finalized = false;
+
+ return 0;
+}
+EXPORT_SYMBOL(cmdq_rec_reset);
+
+int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr)
+{
+ return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
+}
+EXPORT_SYMBOL(cmdq_rec_write);
+
+int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
+ u32 addr, u32 mask)
+{
+ int ret;
+
+ if (mask != 0xffffffff) {
+ ret = cmdq_rec_append_command(handle, CMDQ_CODE_MOVE, 0, ~mask);
+ if (ret)
+ return ret;
+
+ addr = addr | CMDQ_ENABLE_MASK;
+ }
+
+ return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
+}
+EXPORT_SYMBOL(cmdq_rec_write_mask);
+
+int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event)
+{
+ if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
+ event < 0)
+ return -EINVAL;
+
+ return cmdq_rec_append_command(handle, CMDQ_CODE_WFE, event, 0);
+}
+EXPORT_SYMBOL(cmdq_rec_wait);
+
+int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event)
+{
+ if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
+ event < 0)
+ return -EINVAL;
+
+ return cmdq_rec_append_command(handle, CMDQ_CODE_CLEAR_EVENT, event, 0);
+}
+EXPORT_SYMBOL(cmdq_rec_clear_event);
+
+static int cmdq_rec_finalize_command(struct cmdq_rec *handle)
+{
+ int status;
+ struct device *dev;
+ u32 arg_b;
+
+ dev = handle->cqctx->dev;
+
+ if (!handle->finalized) {
+ /* insert EOC and generate IRQ for each command iteration */
+ arg_b = CMDQ_EOC_IRQ_EN;
+ status = cmdq_rec_append_command(handle, CMDQ_CODE_EOC,
+ 0, arg_b);
+ if (status)
+ return status;
+
+ /* JUMP to begin */
+ status = cmdq_rec_append_command(handle, CMDQ_CODE_JUMP, 0, 8);
+ if (status)
+ return status;
+
+ handle->finalized = true;
+ }
+
+ return 0;
+}
+
+static int cmdq_rec_fill_cmd_desc(struct cmdq_rec *handle,
+ struct cmdq_command *desc)
+{
+ int ret;
+
+ ret = cmdq_rec_finalize_command(handle);
+ if (ret)
+ return ret;
+
+ desc->cqctx = handle->cqctx;
+ desc->engine_flag = handle->engine_flag;
+ desc->va_base = handle->buf_ptr;
+ desc->block_size = handle->block_size;
+
+ return ret;
+}
+
+int cmdq_rec_flush(struct cmdq_rec *handle)
+{
+ int ret;
+ struct cmdq_command desc;
+
+ ret = cmdq_rec_fill_cmd_desc(handle, &desc);
+ if (ret)
+ return ret;
+
+ return cmdq_core_submit_task(&desc);
+}
+EXPORT_SYMBOL(cmdq_rec_flush);
+
+static int cmdq_rec_flush_async_cb(struct cmdq_rec *handle,
+ cmdq_async_flush_cb isr_cb,
+ void *isr_data,
+ cmdq_async_flush_cb done_cb,
+ void *done_data)
+{
+ int ret;
+ struct cmdq_command desc;
+ struct cmdq_task *task;
+ struct cmdq_task_cb cb;
+
+ ret = cmdq_rec_fill_cmd_desc(handle, &desc);
+ if (ret)
+ return ret;
+
+ cb.isr_cb = isr_cb;
+ cb.isr_data = isr_data;
+ cb.done_cb = done_cb;
+ cb.done_data = done_data;
+
+ ret = cmdq_core_submit_task_async(&desc, &task, &cb);
+ if (ret)
+ return ret;
+
+ ret = cmdq_core_auto_release_task(task);
+
+ return ret;
+}
+
+int cmdq_rec_flush_async(struct cmdq_rec *handle)
+{
+ return cmdq_rec_flush_async_cb(handle, NULL, NULL, NULL, NULL);
+}
+EXPORT_SYMBOL(cmdq_rec_flush_async);
+
+int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
+ cmdq_async_flush_cb isr_cb,
+ void *isr_data,
+ cmdq_async_flush_cb done_cb,
+ void *done_data)
+{
+ return cmdq_rec_flush_async_cb(handle, isr_cb, isr_data,
+ done_cb, done_data);
+}
+EXPORT_SYMBOL(cmdq_rec_flush_async_callback);
+
+void cmdq_rec_destroy(struct cmdq_rec *handle)
+{
+ if (handle->running_task_ptr)
+ cmdq_rec_stop_running_task(handle);
+
+ /* free command buffer */
+ kfree(handle->buf_ptr);
+ handle->buf_ptr = NULL;
+
+ /* free command handle */
+ kfree(handle);
+}
+EXPORT_SYMBOL(cmdq_rec_destroy);
+
+static int cmdq_probe(struct platform_device *pdev)
+{
+ struct cmdq *cqctx;
+ int ret;
+
+ /* init cmdq context, and save it */
+ ret = cmdq_core_initialize(pdev, &cqctx);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to init cmdq context\n");
+ return ret;
+ }
+ platform_set_drvdata(pdev, cqctx);
+
+ ret = devm_request_irq(&pdev->dev, cqctx->irq, cmdq_irq_handler,
+ IRQF_TRIGGER_LOW | IRQF_SHARED,
+ CMDQ_DRIVER_DEVICE_NAME, cqctx);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register ISR (%d)\n", ret);
+ goto fail;
+ }
+
+ cqctx->clock = devm_clk_get(&pdev->dev, CMDQ_CLK_NAME);
+ if (IS_ERR(cqctx->clock)) {
+ dev_err(&pdev->dev, "failed to get clk:%s\n", CMDQ_CLK_NAME);
+ ret = PTR_ERR(cqctx->clock);
+ goto fail;
+ }
+
+ return ret;
+
+fail:
+ cmdq_core_deinitialize(pdev);
+ return ret;
+}
+
+static int cmdq_remove(struct platform_device *pdev)
+{
+ cmdq_core_deinitialize(pdev);
+ return 0;
+}
+
+static const struct of_device_id cmdq_of_ids[] = {
+ {.compatible = "mediatek,mt8173-gce",},
+ {}
+};
+
+static struct platform_driver cmdq_drv = {
+ .probe = cmdq_probe,
+ .remove = cmdq_remove,
+ .driver = {
+ .name = CMDQ_DRIVER_DEVICE_NAME,
+ .owner = THIS_MODULE,
+ .of_match_table = cmdq_of_ids,
+ }
+};
+
+builtin_platform_driver(cmdq_drv);
diff --git a/include/soc/mediatek/cmdq.h b/include/soc/mediatek/cmdq.h
new file mode 100644
index 0000000..29931c9
--- /dev/null
+++ b/include/soc/mediatek/cmdq.h
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2015 MediaTek Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __MTK_CMDQ_H__
+#define __MTK_CMDQ_H__
+
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+enum cmdq_eng {
+ CMDQ_ENG_DISP_AAL,
+ CMDQ_ENG_DISP_COLOR0,
+ CMDQ_ENG_DISP_COLOR1,
+ CMDQ_ENG_DISP_DPI0,
+ CMDQ_ENG_DISP_DSI0,
+ CMDQ_ENG_DISP_DSI1,
+ CMDQ_ENG_DISP_GAMMA,
+ CMDQ_ENG_DISP_OD,
+ CMDQ_ENG_DISP_OVL0,
+ CMDQ_ENG_DISP_OVL1,
+ CMDQ_ENG_DISP_PWM0,
+ CMDQ_ENG_DISP_PWM1,
+ CMDQ_ENG_DISP_RDMA0,
+ CMDQ_ENG_DISP_RDMA1,
+ CMDQ_ENG_DISP_RDMA2,
+ CMDQ_ENG_DISP_UFOE,
+ CMDQ_ENG_DISP_WDMA0,
+ CMDQ_ENG_DISP_WDMA1,
+ CMDQ_ENG_MAX,
+};
+
+/* events for CMDQ and display */
+enum cmdq_event {
+ /* Display start of frame(SOF) events */
+ CMDQ_EVENT_DISP_OVL0_SOF = 11,
+ CMDQ_EVENT_DISP_OVL1_SOF = 12,
+ CMDQ_EVENT_DISP_RDMA0_SOF = 13,
+ CMDQ_EVENT_DISP_RDMA1_SOF = 14,
+ CMDQ_EVENT_DISP_RDMA2_SOF = 15,
+ CMDQ_EVENT_DISP_WDMA0_SOF = 16,
+ CMDQ_EVENT_DISP_WDMA1_SOF = 17,
+ /* Display end of frame(EOF) events */
+ CMDQ_EVENT_DISP_OVL0_EOF = 39,
+ CMDQ_EVENT_DISP_OVL1_EOF = 40,
+ CMDQ_EVENT_DISP_RDMA0_EOF = 41,
+ CMDQ_EVENT_DISP_RDMA1_EOF = 42,
+ CMDQ_EVENT_DISP_RDMA2_EOF = 43,
+ CMDQ_EVENT_DISP_WDMA0_EOF = 44,
+ CMDQ_EVENT_DISP_WDMA1_EOF = 45,
+ /* Mutex end of frame(EOF) events */
+ CMDQ_EVENT_MUTEX0_STREAM_EOF = 53,
+ CMDQ_EVENT_MUTEX1_STREAM_EOF = 54,
+ CMDQ_EVENT_MUTEX2_STREAM_EOF = 55,
+ CMDQ_EVENT_MUTEX3_STREAM_EOF = 56,
+ CMDQ_EVENT_MUTEX4_STREAM_EOF = 57,
+ /* Display underrun events */
+ CMDQ_EVENT_DISP_RDMA0_UNDERRUN = 63,
+ CMDQ_EVENT_DISP_RDMA1_UNDERRUN = 64,
+ CMDQ_EVENT_DISP_RDMA2_UNDERRUN = 65,
+ /* Keep this at the end of HW events */
+ CMDQ_MAX_HW_EVENT_COUNT = 260,
+ /* This is max event and also can be used as mask. */
+ CMDQ_SYNC_TOKEN_MAX = 0x1ff,
+ /* Invalid event */
+ CMDQ_SYNC_TOKEN_INVALID = -1,
+};
+
+/* called after isr done or task done */
+typedef int (*cmdq_async_flush_cb)(void *data);
+
+struct cmdq_task;
+struct cmdq;
+
+struct cmdq_rec {
+ struct cmdq *cqctx;
+ u64 engine_flag;
+ size_t block_size; /* command size */
+ void *buf_ptr;
+ size_t buf_size;
+ /* running task after flush */
+ struct cmdq_task *running_task_ptr;
+ bool finalized;
+};
+
+/**
+ * cmdq_rec_create() - create command queue record handle
+ * @dev: device
+ * @engine_flag: command queue engine flag
+ * @handle_ptr: command queue record handle pointer to retrieve cmdq_rec
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_create(struct device *dev, u64 engine_flag,
+ struct cmdq_rec **handle_ptr);
+
+/**
+ * cmdq_rec_reset() - reset command queue record commands
+ * @handle: the command queue record handle
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_reset(struct cmdq_rec *handle);
+
+/**
+ * cmdq_rec_write() - append write command to the command queue record
+ * @handle: the command queue record handle
+ * @value: the specified target register value
+ * @addr: the specified target register physical address
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr);
+
+/**
+ * cmdq_rec_write_mask() - append write command with mask to the command queue
+ * record
+ * @handle: the command queue record handle
+ * @value: the specified target register value
+ * @addr: the specified target register physical address
+ * @mask: the specified target register mask
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
+ u32 addr, u32 mask);
+
+/**
+ * cmdq_rec_wait() - append wait command to the command queue record
+ * @handle: the command queue record handle
+ * @event: the desired event type to "wait and CLEAR"
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event);
+
+/**
+ * cmdq_rec_clear_event() - append clear event command to the command queue
+ * record
+ * @handle: the command queue record handle
+ * @event: the desired event to be cleared
+ *
+ * Return: 0 for success; else the error code is returned
+ */
+int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event);
+
+/**
+ * cmdq_rec_flush() - trigger CMDQ to execute the recorded commands
+ * @handle: the command queue record handle
+ *
+ * Return: 0 for success; else the error code is returned
+ *
+ * Trigger CMDQ to execute the recorded commands. Note that this is a
+ * synchronous flush function. When the function returned, the recorded
+ * commands have been done.
+ */
+int cmdq_rec_flush(struct cmdq_rec *handle);
+
+/**
+ * cmdq_rec_flush_async() - trigger CMDQ to asynchronously execute the
+ * recorded commands
+ * @handle: the command queue record handle
+ *
+ * Return: 0 for successfully start execution; else the error code is returned
+ *
+ * Trigger CMDQ to asynchronously execute the recorded commands. Note that this
+ * is an ASYNC function. When the function returned, it may or may not be
+ * finished. There is no way to retrieve the result.
+ */
+int cmdq_rec_flush_async(struct cmdq_rec *handle);
+
+/**
+ * cmdq_rec_flush_async_callback() - trigger CMDQ to asynchronously execute
+ * the recorded commands and call back after
+ * ISR is finished and this flush is finished
+ * @handle: the command queue record handle
+ * @isr_cb: called by ISR in the end of CMDQ ISR
+ * @isr_data: this data will pass back to isr_cb
+ * @done_cb: called after flush is done
+ * @done_data: this data will pass back to done_cb
+ *
+ * Return: 0 for success; else the error code is returned
+ *
+ * Trigger CMDQ to asynchronously execute the recorded commands and call back
+ * after ISR is finished and this flush is finished. Note that this is an ASYNC
+ * function. When the function returned, it may or may not be finished. The ISR
+ * callback function is called in the end of ISR, and the done callback
+ * function is called after all commands are done.
+ */
+int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
+ cmdq_async_flush_cb isr_cb,
+ void *isr_data,
+ cmdq_async_flush_cb done_cb,
+ void *done_data);
+
+/**
+ * cmdq_rec_destroy() - destroy command queue record handle
+ * @handle: the command queue record handle
+ */
+void cmdq_rec_destroy(struct cmdq_rec *handle);
+
+#endif /* __MTK_CMDQ_H__ */
--
2.7.4
More information about the Linux-mediatek
mailing list