[RFC 01/11] soc: mediatek: MediaTek Command Queue (CMDQ) driver
CK Hu
ck.hu at mediatek.com
Mon Oct 2 03:24:49 PDT 2017
Hi, Ulrich:
This patch looks old-fashioned. Please follow up with [1].
[1]
http://lists.infradead.org/pipermail/linux-mediatek/2017-January/007654.html
Regards,
CK
On Fri, 2017-09-29 at 15:09 +0200, Ulrich Hecht wrote:
> Ported from chromeos-3.18 kernel.
>
> Signed-off-by: Ulrich Hecht <ulrich.hecht+renesas at gmail.com>
> ---
> drivers/soc/mediatek/Kconfig | 10 +
> drivers/soc/mediatek/Makefile | 1 +
> drivers/soc/mediatek/mtk-cmdq.c | 2814 +++++++++++++++++++++++++++++++++++++++
> include/soc/mediatek/cmdq.h | 211 +++
> 4 files changed, 3036 insertions(+)
> create mode 100644 drivers/soc/mediatek/mtk-cmdq.c
> create mode 100644 include/soc/mediatek/cmdq.h
>
> diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
> index 609bb34..ef271e0 100644
> --- a/drivers/soc/mediatek/Kconfig
> +++ b/drivers/soc/mediatek/Kconfig
> @@ -1,6 +1,16 @@
> #
> # MediaTek SoC drivers
> #
> +config MTK_CMDQ
> + bool "MediaTek CMDQ Support"
> + depends on ARCH_MEDIATEK || COMPILE_TEST
> + select MTK_INFRACFG
> + help
> + Say yes here to add support for the MediaTek Command Queue (CMDQ)
> + driver. The CMDQ is used to help read/write registers with critical
> + time limitation, such as updating display configuration during the
> + vblank.
> +
> config MTK_INFRACFG
> bool "MediaTek INFRACFG Support"
> depends on ARCH_MEDIATEK || COMPILE_TEST
> diff --git a/drivers/soc/mediatek/Makefile b/drivers/soc/mediatek/Makefile
> index 12998b0..f7397ef 100644
> --- a/drivers/soc/mediatek/Makefile
> +++ b/drivers/soc/mediatek/Makefile
> @@ -1,3 +1,4 @@
> +obj-$(CONFIG_MTK_CMDQ) += mtk-cmdq.o
> obj-$(CONFIG_MTK_INFRACFG) += mtk-infracfg.o
> obj-$(CONFIG_MTK_PMIC_WRAP) += mtk-pmic-wrap.o
> obj-$(CONFIG_MTK_SCPSYS) += mtk-scpsys.o
> diff --git a/drivers/soc/mediatek/mtk-cmdq.c b/drivers/soc/mediatek/mtk-cmdq.c
> new file mode 100644
> index 0000000..a8bfb5c
> --- /dev/null
> +++ b/drivers/soc/mediatek/mtk-cmdq.c
> @@ -0,0 +1,2814 @@
> +/*
> + * Copyright (c) 2015 MediaTek Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <linux/clk.h>
> +#include <linux/clk-provider.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/errno.h>
> +#include <linux/interrupt.h>
> +#include <linux/iopoll.h>
> +#include <linux/kernel.h>
> +#include <linux/kthread.h>
> +#include <linux/ktime.h>
> +#include <linux/module.h>
> +#include <linux/mutex.h>
> +#include <linux/of_irq.h>
> +#include <linux/platform_device.h>
> +#include <linux/slab.h>
> +#include <linux/spinlock.h>
> +#include <linux/workqueue.h>
> +#include <soc/mediatek/cmdq.h>
> +
> +/*
> + * Please calculate this value for each platform.
> + * task number = vblank time / ((task cmds * cmd ticks) / GCE freq)
> + */
> +#define CMDQ_MAX_TASK_IN_THREAD 70
> +
> +#define CMDQ_INITIAL_CMD_BLOCK_SIZE PAGE_SIZE
> +#define CMDQ_CMD_BUF_POOL_BUF_SIZE PAGE_SIZE
> +#define CMDQ_CMD_BUF_POOL_BUF_NUM 140 /* 2 * 70 = 140 */
> +#define CMDQ_INST_SIZE 8 /* instruction is 64-bit */
> +
> +/*
> + * cmdq_thread cookie value is from 0 to CMDQ_MAX_COOKIE_VALUE.
> + * And, this value also be used as MASK.
> + */
> +#define CMDQ_MAX_COOKIE_VALUE 0xffff
> +#define CMDQ_COOKIE_MASK CMDQ_MAX_COOKIE_VALUE
> +
> +#define CMDQ_DEFAULT_TIMEOUT_MS 1000
> +#define CMDQ_ACQUIRE_THREAD_TIMEOUT_MS 5000
> +#define CMDQ_PREALARM_TIMEOUT_NS 200000000
> +
> +#define CMDQ_INVALID_THREAD -1
> +
> +#define CMDQ_DRIVER_DEVICE_NAME "mtk_cmdq"
> +
> +#define CMDQ_CLK_NAME "gce"
> +
> +#define CMDQ_CURR_IRQ_STATUS_OFFSET 0x010
> +#define CMDQ_CURR_LOADED_THR_OFFSET 0x018
> +#define CMDQ_THR_SLOT_CYCLES_OFFSET 0x030
> +#define CMDQ_THR_EXEC_CYCLES_OFFSET 0x034
> +#define CMDQ_THR_TIMEOUT_TIMER_OFFSET 0x038
> +#define CMDQ_BUS_CONTROL_TYPE_OFFSET 0x040
> +
> +#define CMDQ_SYNC_TOKEN_ID_OFFSET 0x060
> +#define CMDQ_SYNC_TOKEN_VAL_OFFSET 0x064
> +#define CMDQ_SYNC_TOKEN_UPD_OFFSET 0x068
> +
> +#define CMDQ_GPR_SHIFT 0x004
> +#define CMDQ_GPR_OFFSET 0x080
> +
> +#define CMDQ_THR_SHIFT 0x080
> +#define CMDQ_THR_WARM_RESET_OFFSET 0x100
> +#define CMDQ_THR_ENABLE_TASK_OFFSET 0x104
> +#define CMDQ_THR_SUSPEND_TASK_OFFSET 0x108
> +#define CMDQ_THR_CURR_STATUS_OFFSET 0x10c
> +#define CMDQ_THR_IRQ_STATUS_OFFSET 0x110
> +#define CMDQ_THR_IRQ_ENABLE_OFFSET 0x114
> +#define CMDQ_THR_CURR_ADDR_OFFSET 0x120
> +#define CMDQ_THR_END_ADDR_OFFSET 0x124
> +#define CMDQ_THR_EXEC_CNT_OFFSET 0x128
> +#define CMDQ_THR_WAIT_TOKEN_OFFSET 0x130
> +#define CMDQ_THR_CFG_OFFSET 0x140
> +#define CMDQ_THR_INST_CYCLES_OFFSET 0x150
> +#define CMDQ_THR_INST_THRESX_OFFSET 0x154
> +#define CMDQ_THR_STATUS_OFFSET 0x18c
> +
> +#define CMDQ_SYNC_TOKEN_SET BIT(16)
> +#define CMDQ_IRQ_MASK 0xffff
> +
> +#define CMDQ_THR_ENABLED 0x1
> +#define CMDQ_THR_DISABLED 0x0
> +#define CMDQ_THR_SUSPEND 0x1
> +#define CMDQ_THR_RESUME 0x0
> +#define CMDQ_THR_STATUS_SUSPENDED BIT(1)
> +#define CMDQ_THR_WARM_RESET BIT(0)
> +#define CMDQ_THR_SLOT_CYCLES 0x3200
> +#define CMDQ_THR_NO_TIMEOUT 0x0
> +#define CMDQ_THR_PRIORITY 3
> +#define CMDQ_THR_IRQ_DONE 0x1
> +#define CMDQ_THR_IRQ_ERROR 0x12
> +#define CMDQ_THR_IRQ_EN 0x13 /* done + error */
> +#define CMDQ_THR_IRQ_MASK 0x13
> +#define CMDQ_THR_EXECUTING BIT(31)
> +#define CMDQ_THR_IS_WAITING BIT(31)
> +
> +#define CMDQ_ARG_A_MASK 0xffffff
> +#define CMDQ_ARG_A_WRITE_MASK 0xffff
> +#define CMDQ_ARG_A_SUBSYS_MASK 0x1f0000
> +#define CMDQ_SUBSYS_MASK 0x1f
> +
> +#define CMDQ_OP_CODE_SHIFT 24
> +#define CMDQ_SUBSYS_SHIFT 16
> +
> +#define CMDQ_JUMP_BY_OFFSET 0x10000000
> +#define CMDQ_JUMP_BY_PA 0x10000001
> +#define CMDQ_JUMP_PASS CMDQ_INST_SIZE
> +
> +#define CMDQ_WFE_UPDATE BIT(31)
> +#define CMDQ_WFE_WAIT BIT(15)
> +#define CMDQ_WFE_WAIT_VALUE 0x1
> +
> +#define CMDQ_MARK_NON_SUSPENDABLE BIT(21) /* 53 - 32 = 21 */
> +#define CMDQ_MARK_NOT_ADD_COUNTER BIT(16) /* 48 - 32 = 16 */
> +#define CMDQ_MARK_PREFETCH_MARKER BIT(20)
> +#define CMDQ_MARK_PREFETCH_MARKER_EN BIT(17)
> +#define CMDQ_MARK_PREFETCH_EN BIT(16)
> +
> +#define CMDQ_EOC_IRQ_EN BIT(0)
> +
> +#define CMDQ_ENABLE_MASK BIT(0)
> +
> +#define CMDQ_OP_CODE_MASK 0xff000000
> +
> +enum cmdq_thread_index {
> + CMDQ_THR_DISP_DSI0 = 0, /* main: dsi0 */
> + CMDQ_THR_DISP_DPI0, /* sub: dpi0 */
> + CMDQ_MAX_THREAD_COUNT, /* max */
> +};
> +
> +struct cmdq_command {
> + struct cmdq *cqctx;
> + /* bit flag of used engines */
> + u64 engine_flag;
> + /*
> + * pointer of instruction buffer
> + * This must point to an 64-bit aligned u32 array
> + */
> + u32 *va_base;
> + /* size of instruction buffer, in bytes. */
> + size_t block_size;
> +};
> +
> +enum cmdq_code {
> + /* These are actual HW op code. */
> + CMDQ_CODE_MOVE = 0x02,
> + CMDQ_CODE_WRITE = 0x04,
> + CMDQ_CODE_JUMP = 0x10,
> + CMDQ_CODE_WFE = 0x20, /* wait for event (and clear) */
> + CMDQ_CODE_CLEAR_EVENT = 0x21, /* clear event */
> + CMDQ_CODE_EOC = 0x40, /* end of command */
> +};
> +
> +enum cmdq_task_state {
> + TASK_STATE_IDLE, /* free task */
> + TASK_STATE_BUSY, /* task running on a thread */
> + TASK_STATE_KILLED, /* task process being killed */
> + TASK_STATE_ERROR, /* task execution error */
> + TASK_STATE_DONE, /* task finished */
> + TASK_STATE_WAITING, /* allocated but waiting for available thread */
> +};
> +
> +struct cmdq_cmd_buf {
> + atomic_t used;
> + void *va;
> + dma_addr_t pa;
> +};
> +
> +struct cmdq_task_cb {
> + /* called by isr */
> + cmdq_async_flush_cb isr_cb;
> + void *isr_data;
> + /* called by releasing task */
> + cmdq_async_flush_cb done_cb;
> + void *done_data;
> +};
> +
> +struct cmdq_task {
> + struct cmdq *cqctx;
> + struct list_head list_entry;
> +
> + /* state for task life cycle */
> + enum cmdq_task_state task_state;
> + /* virtual address of command buffer */
> + u32 *va_base;
> + /* physical address of command buffer */
> + dma_addr_t mva_base;
> + /* size of allocated command buffer */
> + size_t buf_size;
> + /* It points to a cmdq_cmd_buf if this task use command buffer pool. */
> + struct cmdq_cmd_buf *cmd_buf;
> +
> + u64 engine_flag;
> + size_t command_size;
> + u32 num_cmd; /* 2 * number of commands */
> + int reorder;
> + /* HW thread ID; CMDQ_INVALID_THREAD if not running */
> + int thread;
> + /* flag of IRQ received */
> + int irq_flag;
> + /* callback functions */
> + struct cmdq_task_cb cb;
> + /* work item when auto release is used */
> + struct work_struct auto_release_work;
> +
> + ktime_t submit; /* submit time */
> +
> + pid_t caller_pid;
> + char caller_name[TASK_COMM_LEN];
> +};
> +
> +struct cmdq_thread {
> + u32 task_count;
> + u32 wait_cookie;
> + u32 next_cookie;
> + struct cmdq_task *cur_task[CMDQ_MAX_TASK_IN_THREAD];
> +};
> +
> +struct cmdq {
> + struct device *dev;
> +
> + void __iomem *base;
> + u32 irq;
> +
> + /*
> + * task information
> + * task_cache: struct cmdq_task object cache
> + * task_free_list: unused free tasks
> + * task_active_list: active tasks
> + * task_consume_wait_queue_item: task consumption work item
> + * task_auto_release_wq: auto-release workqueue
> + * task_consume_wq: task consumption workqueue (for queued tasks)
> + */
> + struct kmem_cache *task_cache;
> + struct list_head task_free_list;
> + struct list_head task_active_list;
> + struct list_head task_wait_list;
> + struct work_struct task_consume_wait_queue_item;
> + struct workqueue_struct *task_auto_release_wq;
> + struct workqueue_struct *task_consume_wq;
> +
> + struct cmdq_thread thread[CMDQ_MAX_THREAD_COUNT];
> +
> + /* mutex, spinlock, flag */
> + struct mutex task_mutex; /* for task list */
> + struct mutex clock_mutex; /* for clock operation */
> + spinlock_t thread_lock; /* for cmdq hardware thread */
> + int thread_usage;
> + spinlock_t exec_lock; /* for exec task */
> +
> + /* command buffer pool */
> + struct cmdq_cmd_buf cmd_buf_pool[CMDQ_CMD_BUF_POOL_BUF_NUM];
> +
> + /*
> + * notification
> + * wait_queue: for task done
> + * thread_dispatch_queue: for thread acquiring
> + */
> + wait_queue_head_t wait_queue[CMDQ_MAX_THREAD_COUNT];
> + wait_queue_head_t thread_dispatch_queue;
> +
> + /* ccf */
> + struct clk *clock;
> +};
> +
> +struct cmdq_event_item {
> + enum cmdq_event event;
> + const char *name;
> +};
> +
> +struct cmdq_subsys {
> + u32 base_addr;
> + int id;
> + const char *name;
> +};
> +
> +static const struct cmdq_event_item cmdq_events[] = {
> + /* Display start of frame(SOF) events */
> + {CMDQ_EVENT_DISP_OVL0_SOF, "CMDQ_EVENT_DISP_OVL0_SOF"},
> + {CMDQ_EVENT_DISP_OVL1_SOF, "CMDQ_EVENT_DISP_OVL1_SOF"},
> + {CMDQ_EVENT_DISP_RDMA0_SOF, "CMDQ_EVENT_DISP_RDMA0_SOF"},
> + {CMDQ_EVENT_DISP_RDMA1_SOF, "CMDQ_EVENT_DISP_RDMA1_SOF"},
> + {CMDQ_EVENT_DISP_RDMA2_SOF, "CMDQ_EVENT_DISP_RDMA2_SOF"},
> + {CMDQ_EVENT_DISP_WDMA0_SOF, "CMDQ_EVENT_DISP_WDMA0_SOF"},
> + {CMDQ_EVENT_DISP_WDMA1_SOF, "CMDQ_EVENT_DISP_WDMA1_SOF"},
> + /* Display end of frame(EOF) events */
> + {CMDQ_EVENT_DISP_OVL0_EOF, "CMDQ_EVENT_DISP_OVL0_EOF"},
> + {CMDQ_EVENT_DISP_OVL1_EOF, "CMDQ_EVENT_DISP_OVL1_EOF"},
> + {CMDQ_EVENT_DISP_RDMA0_EOF, "CMDQ_EVENT_DISP_RDMA0_EOF"},
> + {CMDQ_EVENT_DISP_RDMA1_EOF, "CMDQ_EVENT_DISP_RDMA1_EOF"},
> + {CMDQ_EVENT_DISP_RDMA2_EOF, "CMDQ_EVENT_DISP_RDMA2_EOF"},
> + {CMDQ_EVENT_DISP_WDMA0_EOF, "CMDQ_EVENT_DISP_WDMA0_EOF"},
> + {CMDQ_EVENT_DISP_WDMA1_EOF, "CMDQ_EVENT_DISP_WDMA1_EOF"},
> + /* Mutex end of frame(EOF) events */
> + {CMDQ_EVENT_MUTEX0_STREAM_EOF, "CMDQ_EVENT_MUTEX0_STREAM_EOF"},
> + {CMDQ_EVENT_MUTEX1_STREAM_EOF, "CMDQ_EVENT_MUTEX1_STREAM_EOF"},
> + {CMDQ_EVENT_MUTEX2_STREAM_EOF, "CMDQ_EVENT_MUTEX2_STREAM_EOF"},
> + {CMDQ_EVENT_MUTEX3_STREAM_EOF, "CMDQ_EVENT_MUTEX3_STREAM_EOF"},
> + {CMDQ_EVENT_MUTEX4_STREAM_EOF, "CMDQ_EVENT_MUTEX4_STREAM_EOF"},
> + /* Display underrun events */
> + {CMDQ_EVENT_DISP_RDMA0_UNDERRUN, "CMDQ_EVENT_DISP_RDMA0_UNDERRUN"},
> + {CMDQ_EVENT_DISP_RDMA1_UNDERRUN, "CMDQ_EVENT_DISP_RDMA1_UNDERRUN"},
> + {CMDQ_EVENT_DISP_RDMA2_UNDERRUN, "CMDQ_EVENT_DISP_RDMA2_UNDERRUN"},
> + /* Keep this at the end of HW events */
> + {CMDQ_MAX_HW_EVENT_COUNT, "CMDQ_MAX_HW_EVENT_COUNT"},
> + /* This is max event and also can be used as mask. */
> + {CMDQ_SYNC_TOKEN_MAX, "CMDQ_SYNC_TOKEN_MAX"},
> + /* Invalid event */
> + {CMDQ_SYNC_TOKEN_INVALID, "CMDQ_SYNC_TOKEN_INVALID"},
> +};
> +
> +static const struct cmdq_subsys g_subsys[] = {
> + {0x1400, 1, "MMSYS"},
> + {0x1401, 2, "DISP"},
> + {0x1402, 3, "DISP"},
> +};
> +
> +static const char *cmdq_event_get_name(enum cmdq_event event)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(cmdq_events); i++)
> + if (cmdq_events[i].event == event)
> + return cmdq_events[i].name;
> +
> + return "CMDQ_EVENT_UNKNOWN";
> +}
> +
> +static void cmdq_event_reset(struct cmdq *cqctx)
> +{
> + int i;
> +
> + /* set all defined HW events to 0 */
> + for (i = 0; i < ARRAY_SIZE(cmdq_events); i++) {
> + if (cmdq_events[i].event >= CMDQ_MAX_HW_EVENT_COUNT)
> + break;
> + writel(cmdq_events[i].event,
> + cqctx->base + CMDQ_SYNC_TOKEN_UPD_OFFSET);
> + }
> +}
> +
> +static int cmdq_subsys_base_addr_to_id(u32 base_addr)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
> + if (g_subsys[i].base_addr == base_addr)
> + return g_subsys[i].id;
> + }
> +
> + return -EFAULT;
> +}
> +
> +static u32 cmdq_subsys_id_to_base_addr(int id)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
> + if (g_subsys[i].id == id)
> + return g_subsys[i].base_addr;
> + }
> +
> + return 0;
> +}
> +
> +static const char *cmdq_subsys_base_addr_to_name(u32 base_addr)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(g_subsys); i++)
> + if (g_subsys[i].base_addr == base_addr)
> + return g_subsys[i].name;
> +
> + return NULL;
> +}
> +
> +static int cmdq_eng_get_thread(u64 flag)
> +{
> + if (flag & BIT_ULL(CMDQ_ENG_DISP_DSI0))
> + return CMDQ_THR_DISP_DSI0;
> + else /* CMDQ_ENG_DISP_DPI0 */
> + return CMDQ_THR_DISP_DPI0;
> +}
> +
> +static const char *cmdq_event_get_module(enum cmdq_event event)
> +{
> + const char *module;
> +
> + switch (event) {
> + case CMDQ_EVENT_DISP_RDMA0_SOF:
> + case CMDQ_EVENT_DISP_RDMA1_SOF:
> + case CMDQ_EVENT_DISP_RDMA2_SOF:
> + case CMDQ_EVENT_DISP_RDMA0_EOF:
> + case CMDQ_EVENT_DISP_RDMA1_EOF:
> + case CMDQ_EVENT_DISP_RDMA2_EOF:
> + case CMDQ_EVENT_DISP_RDMA0_UNDERRUN:
> + case CMDQ_EVENT_DISP_RDMA1_UNDERRUN:
> + case CMDQ_EVENT_DISP_RDMA2_UNDERRUN:
> + module = "DISP_RDMA";
> + break;
> + case CMDQ_EVENT_DISP_WDMA0_SOF:
> + case CMDQ_EVENT_DISP_WDMA1_SOF:
> + case CMDQ_EVENT_DISP_WDMA0_EOF:
> + case CMDQ_EVENT_DISP_WDMA1_EOF:
> + module = "DISP_WDMA";
> + break;
> + case CMDQ_EVENT_DISP_OVL0_SOF:
> + case CMDQ_EVENT_DISP_OVL1_SOF:
> + case CMDQ_EVENT_DISP_OVL0_EOF:
> + case CMDQ_EVENT_DISP_OVL1_EOF:
> + module = "DISP_OVL";
> + break;
> + case CMDQ_EVENT_MUTEX0_STREAM_EOF ... CMDQ_EVENT_MUTEX4_STREAM_EOF:
> + module = "DISP";
> + break;
> + default:
> + module = "CMDQ";
> + break;
> + }
> +
> + return module;
> +}
> +
> +static u32 cmdq_thread_get_cookie(struct cmdq *cqctx, int tid)
> +{
> + return readl(cqctx->base + CMDQ_THR_EXEC_CNT_OFFSET +
> + CMDQ_THR_SHIFT * tid) & CMDQ_COOKIE_MASK;
> +}
> +
> +static int cmdq_cmd_buf_pool_init(struct cmdq *cqctx)
> +{
> + struct device *dev = cqctx->dev;
> + int i;
> + int ret = 0;
> + struct cmdq_cmd_buf *buf;
> +
> + for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> + buf = &cqctx->cmd_buf_pool[i];
> + buf->va = dma_alloc_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE,
> + &buf->pa, GFP_KERNEL);
> + if (!buf->va) {
> + dev_err(dev, "failed to alloc cmdq_cmd_buf\n");
> + ret = -ENOMEM;
> + goto fail_alloc;
> + }
> + }
> +
> + return 0;
> +
> +fail_alloc:
> + for (i -= 1; i >= 0 ; i--) {
> + buf = &cqctx->cmd_buf_pool[i];
> + dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
> + buf->pa);
> + }
> +
> + return ret;
> +}
> +
> +static void cmdq_cmd_buf_pool_uninit(struct cmdq *cqctx)
> +{
> + struct device *dev = cqctx->dev;
> + int i;
> + struct cmdq_cmd_buf *buf;
> +
> + for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> + buf = &cqctx->cmd_buf_pool[i];
> + dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
> + buf->pa);
> + if (atomic_read(&buf->used))
> + dev_err(dev,
> + "cmdq_cmd_buf[%d] va:0x%p still in use\n",
> + i, buf->va);
> + }
> +}
> +
> +static struct cmdq_cmd_buf *cmdq_cmd_buf_pool_get(struct cmdq *cqctx)
> +{
> + int i;
> + struct cmdq_cmd_buf *buf;
> +
> + for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> + buf = &cqctx->cmd_buf_pool[i];
> + if (!atomic_cmpxchg(&buf->used, 0, 1))
> + return buf;
> + }
> +
> + return NULL;
> +}
> +
> +static void cmdq_cmd_buf_pool_put(struct cmdq_cmd_buf *buf)
> +{
> + atomic_set(&buf->used, 0);
> +}
> +
> +static int cmdq_subsys_from_phys_addr(struct cmdq *cqctx, u32 cmdq_phys_addr)
> +{
> + u32 base_addr = cmdq_phys_addr >> 16;
> + int subsys = cmdq_subsys_base_addr_to_id(base_addr);
> +
> + if (subsys < 0)
> + dev_err(cqctx->dev,
> + "unknown subsys: error=%d, phys=0x%08x\n",
> + subsys, cmdq_phys_addr);
> +
> + return subsys;
> +}
> +
> +/*
> + * It's a kmemcache creator for cmdq_task to initialize variables
> + * without command buffer.
> + */
> +static void cmdq_task_ctor(void *param)
> +{
> + struct cmdq_task *task = param;
> +
> + memset(task, 0, sizeof(*task));
> + INIT_LIST_HEAD(&task->list_entry);
> + task->task_state = TASK_STATE_IDLE;
> + task->thread = CMDQ_INVALID_THREAD;
> +}
> +
> +static void cmdq_task_free_command_buffer(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> +
> + if (!task->va_base)
> + return;
> +
> + if (task->cmd_buf)
> + cmdq_cmd_buf_pool_put(task->cmd_buf);
> + else
> + dma_free_coherent(dev, task->buf_size, task->va_base,
> + task->mva_base);
> +
> + task->va_base = NULL;
> + task->mva_base = 0;
> + task->buf_size = 0;
> + task->command_size = 0;
> + task->num_cmd = 0;
> + task->cmd_buf = NULL;
> +}
> +
> +/*
> + * Ensure size of command buffer in the given cmdq_task.
> + * Existing buffer data will be copied to new buffer.
> + * This buffer is guaranteed to be physically continuous.
> + * returns -ENOMEM if cannot allocate new buffer
> + */
> +static int cmdq_task_realloc_command_buffer(struct cmdq_task *task, size_t size)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + void *new_buf = NULL;
> + dma_addr_t new_mva_base;
> + size_t cmd_size;
> + u32 num_cmd;
> + struct cmdq_cmd_buf *cmd_buf = NULL;
> +
> + if (task->va_base && task->buf_size >= size)
> + return 0;
> +
> + /* try command pool first */
> + if (size <= CMDQ_CMD_BUF_POOL_BUF_SIZE) {
> + cmd_buf = cmdq_cmd_buf_pool_get(cqctx);
> + if (cmd_buf) {
> + new_buf = cmd_buf->va;
> + new_mva_base = cmd_buf->pa;
> + memset(new_buf, 0, CMDQ_CMD_BUF_POOL_BUF_SIZE);
> + }
> + }
> +
> + if (!new_buf) {
> + new_buf = dma_alloc_coherent(dev, size, &new_mva_base,
> + GFP_KERNEL);
> + if (!new_buf) {
> + dev_err(dev, "realloc cmd buffer of size %zu failed\n",
> + size);
> + return -ENOMEM;
> + }
> + }
> +
> + /* copy and release old buffer */
> + if (task->va_base)
> + memcpy(new_buf, task->va_base, task->buf_size);
> +
> + /*
> + * we should keep track of num_cmd and cmd_size
> + * since they are cleared in free command buffer
> + */
> + num_cmd = task->num_cmd;
> + cmd_size = task->command_size;
> + cmdq_task_free_command_buffer(task);
> +
> + /* attach the new buffer */
> + task->va_base = new_buf;
> + task->mva_base = new_mva_base;
> + task->buf_size = cmd_buf ? CMDQ_CMD_BUF_POOL_BUF_SIZE : size;
> + task->num_cmd = num_cmd;
> + task->command_size = cmd_size;
> + task->cmd_buf = cmd_buf;
> +
> + return 0;
> +}
> +
> +/* allocate and initialize struct cmdq_task and its command buffer */
> +static struct cmdq_task *cmdq_task_create(struct cmdq *cqctx)
> +{
> + struct device *dev = cqctx->dev;
> + struct cmdq_task *task;
> + int status;
> +
> + task = kmem_cache_alloc(cqctx->task_cache, GFP_KERNEL);
> + task->cqctx = cqctx;
> + status = cmdq_task_realloc_command_buffer(
> + task, CMDQ_INITIAL_CMD_BLOCK_SIZE);
> + if (status < 0) {
> + dev_err(dev, "allocate command buffer failed\n");
> + kmem_cache_free(cqctx->task_cache, task);
> + return NULL;
> + }
> + return task;
> +}
> +
> +static int cmdq_dev_init(struct platform_device *pdev, struct cmdq *cqctx)
> +{
> + struct device *dev = &pdev->dev;
> + struct device_node *node = dev->of_node;
> + struct resource *res;
> +
> + res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
> + cqctx->base = devm_ioremap_resource(dev, res);
> + if (IS_ERR(cqctx->base)) {
> + dev_err(dev, "failed to ioremap gce\n");
> + return PTR_ERR(cqctx->base);
> + }
> +
> + cqctx->irq = irq_of_parse_and_map(node, 0);
> + if (!cqctx->irq) {
> + dev_err(dev, "failed to get irq\n");
> + return -EINVAL;
> + }
> +
> + dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n",
> + dev, cqctx->base, cqctx->irq);
> + return 0;
> +}
> +
> +static void cmdq_task_release_unlocked(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> +
> + /* This func should be inside cqctx->task_mutex mutex */
> + lockdep_assert_held(&cqctx->task_mutex);
> +
> + task->task_state = TASK_STATE_IDLE;
> + task->thread = CMDQ_INVALID_THREAD;
> +
> + cmdq_task_free_command_buffer(task);
> +
> + /*
> + * move from active/waiting list to free list
> + * todo: shrink free list
> + */
> + list_move_tail(&task->list_entry, &cqctx->task_free_list);
> +}
> +
> +static void cmdq_task_release_internal(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> +
> + mutex_lock(&cqctx->task_mutex);
> + cmdq_task_release_unlocked(task);
> + mutex_unlock(&cqctx->task_mutex);
> +}
> +
> +static struct cmdq_task *cmdq_core_find_free_task(struct cmdq *cqctx)
> +{
> + struct cmdq_task *task;
> +
> + mutex_lock(&cqctx->task_mutex);
> +
> + /*
> + * Pick from free list first;
> + * create one if there is no free entry.
> + */
> + if (list_empty(&cqctx->task_free_list)) {
> + task = cmdq_task_create(cqctx);
> + } else {
> + task = list_first_entry(&cqctx->task_free_list,
> + struct cmdq_task, list_entry);
> + /* remove from free list */
> + list_del_init(&task->list_entry);
> + }
> +
> + mutex_unlock(&cqctx->task_mutex);
> +
> + return task;
> +}
> +
> +/* After dropping error task, we have to reorder remaining valid tasks. */
> +static void cmdq_thread_reorder_task_array(struct cmdq_thread *thread,
> + int prev_id)
> +{
> + int i, j;
> + int next_id, search_id;
> + int reorder_count = 0;
> + struct cmdq_task *task;
> +
> + next_id = prev_id + 1;
> + for (i = 1; i < (CMDQ_MAX_TASK_IN_THREAD - 1); i++, next_id++) {
> + if (next_id >= CMDQ_MAX_TASK_IN_THREAD)
> + next_id = 0;
> +
> + if (thread->cur_task[next_id])
> + break;
> +
> + search_id = next_id + 1;
> + for (j = (i + 1); j < CMDQ_MAX_TASK_IN_THREAD;
> + j++, search_id++) {
> + if (search_id >= CMDQ_MAX_TASK_IN_THREAD)
> + search_id = 0;
> +
> + if (thread->cur_task[search_id]) {
> + thread->cur_task[next_id] =
> + thread->cur_task[search_id];
> + thread->cur_task[search_id] = NULL;
> + if ((j - i) > reorder_count)
> + reorder_count = j - i;
> +
> + break;
> + }
> + }
> +
> + task = thread->cur_task[next_id];
> + if ((task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_OFFSET) &&
> + (task->va_base[task->num_cmd - 2] == CMDQ_JUMP_PASS)) {
> + /* We reached the last task */
> + break;
> + }
> + }
> +
> + thread->next_cookie -= reorder_count;
> +}
> +
> +static int cmdq_core_sync_command(struct cmdq_task *task,
> + struct cmdq_command *cmd_desc)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + int status;
> + size_t size;
> +
> + size = task->command_size + CMDQ_INST_SIZE;
> + status = cmdq_task_realloc_command_buffer(task, size);
> + if (status < 0) {
> + dev_err(dev, "failed to realloc command buffer\n");
> + dev_err(dev, "task=0x%p, request size=%zu\n", task, size);
> + return status;
> + }
> +
> + /* copy the commands to our DMA buffer */
> + memcpy(task->va_base, cmd_desc->va_base, cmd_desc->block_size);
> +
> + /* re-adjust num_cmd according to command_size */
> + task->num_cmd = task->command_size / sizeof(task->va_base[0]);
> +
> + return 0;
> +}
> +
> +static struct cmdq_task *cmdq_core_acquire_task(struct cmdq_command *cmd_desc,
> + struct cmdq_task_cb *cb)
> +{
> + struct cmdq *cqctx = cmd_desc->cqctx;
> + struct device *dev = cqctx->dev;
> + struct cmdq_task *task;
> +
> + task = cmdq_core_find_free_task(cqctx);
> + if (!task) {
> + dev_err(dev, "can't acquire task info\n");
> + return NULL;
> + }
> +
> + /* initialize field values */
> + task->engine_flag = cmd_desc->engine_flag;
> + task->task_state = TASK_STATE_WAITING;
> + task->reorder = 0;
> + task->thread = CMDQ_INVALID_THREAD;
> + task->irq_flag = 0x0;
> + if (cb)
> + task->cb = *cb;
> + else
> + memset(&task->cb, 0, sizeof(task->cb));
> + task->command_size = cmd_desc->block_size;
> +
> + /* store caller info for debug */
> + if (current) {
> + task->caller_pid = current->pid;
> + memcpy(task->caller_name, current->comm, sizeof(current->comm));
> + }
> +
> + if (cmdq_core_sync_command(task, cmd_desc) < 0) {
> + dev_err(dev, "fail to sync command\n");
> + cmdq_task_release_internal(task);
> + return NULL;
> + }
> +
> + /* insert into waiting list to process */
> + if (task) {
> + task->submit = ktime_get();
> + mutex_lock(&cqctx->task_mutex);
> + list_add_tail(&task->list_entry, &cqctx->task_wait_list);
> + mutex_unlock(&cqctx->task_mutex);
> + }
> +
> + return task;
> +}
> +
> +static int cmdq_clk_enable(struct cmdq *cqctx)
> +{
> + struct device *dev = cqctx->dev;
> + int ret = 0;
> +
> + if (cqctx->thread_usage == 0) {
> + ret = clk_prepare_enable(cqctx->clock);
> + if (ret) {
> + dev_err(dev, "prepare and enable clk:%s fail\n",
> + CMDQ_CLK_NAME);
> + return ret;
> + }
> + cmdq_event_reset(cqctx);
> + }
> + cqctx->thread_usage++;
> +
> + return ret;
> +}
> +
> +static void cmdq_clk_disable(struct cmdq *cqctx)
> +{
> + cqctx->thread_usage--;
> + if (cqctx->thread_usage <= 0)
> + clk_disable_unprepare(cqctx->clock);
> +}
> +
> +static int cmdq_core_find_free_thread(struct cmdq *cqctx, int tid)
> +{
> + struct cmdq_thread *thread = cqctx->thread;
> + u32 next_cookie;
> +
> + /*
> + * make sure the found thread has enough space for the task;
> + * cmdq_thread->cur_task has size limitation.
> + */
> + if (thread[tid].task_count >= CMDQ_MAX_TASK_IN_THREAD) {
> + dev_warn(cqctx->dev, "thread(%d) task count = %d\n",
> + tid, thread[tid].task_count);
> + return CMDQ_INVALID_THREAD;
> + }
> +
> + next_cookie = thread[tid].next_cookie % CMDQ_MAX_TASK_IN_THREAD;
> + if (thread[tid].cur_task[next_cookie]) {
> + dev_warn(cqctx->dev, "thread(%d) next cookie = %d\n",
> + tid, next_cookie);
> + return CMDQ_INVALID_THREAD;
> + }
> +
> + return tid;
> +}
> +
> +static struct cmdq_thread *cmdq_core_acquire_thread(struct cmdq *cqctx,
> + int candidate_tid)
> +{
> + int tid;
> +
> + tid = cmdq_core_find_free_thread(cqctx, candidate_tid);
> + if (tid != CMDQ_INVALID_THREAD) {
> + mutex_lock(&cqctx->clock_mutex);
> + cmdq_clk_enable(cqctx);
> + mutex_unlock(&cqctx->clock_mutex);
> + return &cqctx->thread[tid];
> + }
> + return NULL;
> +}
> +
> +static void cmdq_core_release_thread(struct cmdq *cqctx, int tid)
> +{
> + if (WARN_ON(tid == CMDQ_INVALID_THREAD))
> + return;
> +
> + mutex_lock(&cqctx->clock_mutex);
> + cmdq_clk_disable(cqctx);
> + mutex_unlock(&cqctx->clock_mutex);
> +}
> +
> +static void cmdq_task_remove_thread(struct cmdq_task *task)
> +{
> + int tid = task->thread;
> +
> + task->thread = CMDQ_INVALID_THREAD;
> + cmdq_core_release_thread(task->cqctx, tid);
> +}
> +
> +static int cmdq_thread_suspend(struct cmdq *cqctx, int tid)
> +{
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + u32 enabled;
> + u32 status;
> +
> + /* write suspend bit */
> + writel(CMDQ_THR_SUSPEND,
> + gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + /* If already disabled, treat as suspended successful. */
> + enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + if (!(enabled & CMDQ_THR_ENABLED))
> + return 0;
> +
> + /* poll suspended status */
> + if (readl_poll_timeout_atomic(gce_base +
> + CMDQ_THR_CURR_STATUS_OFFSET +
> + CMDQ_THR_SHIFT * tid,
> + status,
> + status & CMDQ_THR_STATUS_SUSPENDED,
> + 0, 10)) {
> + dev_err(dev, "Suspend HW thread %d failed\n", tid);
> + return -EFAULT;
> + }
> +
> + return 0;
> +}
> +
> +static void cmdq_thread_resume(struct cmdq *cqctx, int tid)
> +{
> + void __iomem *gce_base = cqctx->base;
> +
> + writel(CMDQ_THR_RESUME,
> + gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +}
> +
> +static int cmdq_thread_reset(struct cmdq *cqctx, int tid)
> +{
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + u32 warm_reset;
> +
> + writel(CMDQ_THR_WARM_RESET,
> + gce_base + CMDQ_THR_WARM_RESET_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + if (readl_poll_timeout_atomic(gce_base + CMDQ_THR_WARM_RESET_OFFSET +
> + CMDQ_THR_SHIFT * tid,
> + warm_reset,
> + !(warm_reset & CMDQ_THR_WARM_RESET),
> + 0, 10)) {
> + dev_err(dev, "Reset HW thread %d failed\n", tid);
> + return -EFAULT;
> + }
> +
> + writel(CMDQ_THR_SLOT_CYCLES, gce_base + CMDQ_THR_SLOT_CYCLES_OFFSET);
> + return 0;
> +}
> +
> +static int cmdq_thread_disable(struct cmdq *cqctx, int tid)
> +{
> + void __iomem *gce_base = cqctx->base;
> +
> + cmdq_thread_reset(cqctx, tid);
> + writel(CMDQ_THR_DISABLED,
> + gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + return 0;
> +}
> +
> +static u32 *cmdq_task_get_pc_and_inst(const struct cmdq_task *task, int tid,
> + u32 insts[2])
> +{
> + struct cmdq *cqctx;
> + void __iomem *gce_base;
> + unsigned long pc_pa;
> + u8 *pc_va;
> + u8 *cmd_end;
> +
> + memset(insts, 0, sizeof(u32) * 2);
> +
> + if (!task ||
> + !task->va_base ||
> + tid == CMDQ_INVALID_THREAD) {
> + pr_err("cmdq get pc failed since invalid param, task 0x%p, task->va_base:0x%p, thread:%d\n",
> + task, task->va_base, tid);
> + return NULL;
> + }
> +
> + cqctx = task->cqctx;
> + gce_base = cqctx->base;
> +
> + pc_pa = (unsigned long)readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + pc_va = (u8 *)task->va_base + (pc_pa - task->mva_base);
> + cmd_end = (u8 *)(task->va_base + task->num_cmd - 1);
> +
> + if (((u8 *)task->va_base <= pc_va) && (pc_va <= cmd_end)) {
> + if (pc_va < cmd_end) {
> + /* get arg_a and arg_b */
> + insts[0] = readl(pc_va);
> + insts[1] = readl(pc_va + 4);
> + } else {
> + /* get arg_a and arg_b of previous cmd */
> + insts[0] = readl(pc_va - 8);
> + insts[1] = readl(pc_va - 4);
> + }
> + } else {
> + return NULL;
> + }
> +
> + return (u32 *)pc_va;
> +}
> +
> +static const char *cmdq_core_parse_module_from_subsys(u32 arg_a)
> +{
> + int id = (arg_a & CMDQ_ARG_A_SUBSYS_MASK) >> CMDQ_SUBSYS_SHIFT;
> + u32 base_addr = cmdq_subsys_id_to_base_addr(id);
> + const char *module = cmdq_subsys_base_addr_to_name(base_addr);
> +
> + return module ? module : "CMDQ";
> +}
> +
> +static const char *cmdq_core_parse_op(u32 op_code)
> +{
> + switch (op_code) {
> + case CMDQ_CODE_WRITE:
> + return "WRIT";
> + case CMDQ_CODE_WFE:
> + return "SYNC";
> + case CMDQ_CODE_MOVE:
> + return "MASK";
> + case CMDQ_CODE_JUMP:
> + return "JUMP";
> + case CMDQ_CODE_EOC:
> + return "MARK";
> + }
> + return NULL;
> +}
> +
> +static void cmdq_core_parse_error(struct cmdq_task *task, int tid,
> + const char **module_name, int *flag,
> + u32 *inst_a, u32 *inst_b)
> +{
> + int irq_flag = task->irq_flag;
> + u32 insts[2] = { 0 };
> + const char *module;
> +
> + /*
> + * other cases, use instruction to judge
> + * because engine flag are not sufficient
> + */
> + if (cmdq_task_get_pc_and_inst(task, tid, insts)) {
> + u32 op, arg_a, arg_b;
> +
> + op = insts[1] >> CMDQ_OP_CODE_SHIFT;
> + arg_a = insts[1] & CMDQ_ARG_A_MASK;
> + arg_b = insts[0];
> +
> + switch (op) {
> + case CMDQ_CODE_WRITE:
> + module = cmdq_core_parse_module_from_subsys(arg_a);
> + break;
> + case CMDQ_CODE_WFE:
> + /* arg_a is the event id */
> + module = cmdq_event_get_module((enum cmdq_event)arg_a);
> + break;
> + case CMDQ_CODE_MOVE:
> + case CMDQ_CODE_JUMP:
> + case CMDQ_CODE_EOC:
> + default:
> + module = "CMDQ";
> + break;
> + }
> + } else {
> + module = "CMDQ";
> + }
> +
> + /* fill output parameter */
> + *module_name = module;
> + *flag = irq_flag;
> + *inst_a = insts[1];
> + *inst_b = insts[0];
> +}
> +
> +static void cmdq_thread_insert_task_by_cookie(struct cmdq_thread *thread,
> + struct cmdq_task *task,
> + int cookie)
> +{
> + thread->wait_cookie = cookie;
> + thread->next_cookie = cookie + 1;
> + if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
> + thread->next_cookie = 0;
> +
> + /* first task, so set to 1 */
> + thread->task_count = 1;
> +
> + thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
> +}
> +
> +static int cmdq_thread_remove_task_by_index(struct cmdq_thread *thread,
> + int index,
> + enum cmdq_task_state new_state)
> +{
> + struct cmdq_task *task;
> + struct device *dev;
> +
> + task = thread->cur_task[index];
> + if (!task) {
> + pr_err("%s: remove fail, task:%d on thread:0x%p is NULL\n",
> + __func__, index, thread);
> + return -EINVAL;
> + }
> + dev = task->cqctx->dev;
> +
> + /*
> + * note timing to switch a task to done_status(_ERROR, _KILLED, _DONE)
> + * is aligned with thread's taskcount change
> + * check task status to prevent double clean-up thread's taskcount
> + */
> + if (task->task_state != TASK_STATE_BUSY) {
> + dev_err(dev, "remove task failed\n");
> + dev_err(dev, "state:%d. thread:0x%p, task:%d, new_state:%d\n",
> + task->task_state, thread, index, new_state);
> + return -EINVAL;
> + }
> +
> + if (thread->task_count == 0) {
> + dev_err(dev, "no task to remove\n");
> + dev_err(dev, "thread:%d, index:%d\n", task->thread, index);
> + return -EINVAL;
> + }
> +
> + task->task_state = new_state;
> + thread->cur_task[index] = NULL;
> + thread->task_count--;
> +
> + return 0;
> +}
> +
> +static int cmdq_thread_force_remove_task(struct cmdq_task *task, int tid)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + void __iomem *gce_base = cqctx->base;
> + int status;
> + int cookie;
> + struct cmdq_task *exec_task;
> +
> + status = cmdq_thread_suspend(cqctx, tid);
> +
> + writel(CMDQ_THR_NO_TIMEOUT,
> + gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + /* The cookie of the task currently being processed */
> + cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
> +
> + exec_task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
> + if (exec_task && exec_task == task) {
> + dma_addr_t eoc_pa = task->mva_base + task->command_size - 16;
> +
> + /* The task is executed now, set the PC to EOC for bypass */
> + writel(eoc_pa,
> + gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = NULL;
> + task->task_state = TASK_STATE_KILLED;
> + } else {
> + int i, j;
> +
> + j = thread->task_count;
> + for (i = cookie; j > 0; j--, i++) {
> + i %= CMDQ_MAX_TASK_IN_THREAD;
> +
> + exec_task = thread->cur_task[i];
> + if (!exec_task)
> + continue;
> +
> + if ((exec_task->va_base[exec_task->num_cmd - 1] ==
> + CMDQ_JUMP_BY_OFFSET) &&
> + (exec_task->va_base[exec_task->num_cmd - 2] ==
> + CMDQ_JUMP_PASS)) {
> + /* reached the last task */
> + break;
> + }
> +
> + if (exec_task->va_base[exec_task->num_cmd - 2] ==
> + task->mva_base) {
> + /* fake EOC command */
> + exec_task->va_base[exec_task->num_cmd - 2] =
> + CMDQ_EOC_IRQ_EN;
> + exec_task->va_base[exec_task->num_cmd - 1] =
> + CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT;
> +
> + /* bypass the task */
> + exec_task->va_base[exec_task->num_cmd] =
> + task->va_base[task->num_cmd - 2];
> + exec_task->va_base[exec_task->num_cmd + 1] =
> + task->va_base[task->num_cmd - 1];
> +
> + i = (i + 1) % CMDQ_MAX_TASK_IN_THREAD;
> +
> + thread->cur_task[i] = NULL;
> + task->task_state = TASK_STATE_KILLED;
> + status = 0;
> + break;
> + }
> + }
> + }
> +
> + return status;
> +}
> +
> +static struct cmdq_task *cmdq_thread_search_task_by_pc(
> + const struct cmdq_thread *thread, u32 pc)
> +{
> + struct cmdq_task *task;
> + int i;
> +
> + for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> + task = thread->cur_task[i];
> + if (task &&
> + pc >= task->mva_base &&
> + pc <= task->mva_base + task->command_size)
> + break;
> + }
> +
> + return task;
> +}
> +
> +/*
> + * Re-fetch thread's command buffer
> + * Use Case:
> + * If SW modifies command buffer content after SW configed commands to GCE,
> + * SW should notify GCE to re-fetch commands in order to
> + * prevent inconsistent command buffer content between DRAM and GCE's SRAM.
> + */
> +static void cmdq_core_invalidate_hw_fetched_buffer(struct cmdq *cqctx,
> + int tid)
> +{
> + void __iomem *pc_va;
> + u32 pc;
> +
> + /*
> + * Setting HW thread PC will invoke that
> + * GCE (CMDQ HW) gives up fetched command buffer,
> + * and fetch command from DRAM to GCE's SRAM again.
> + */
> + pc_va = cqctx->base + CMDQ_THR_CURR_ADDR_OFFSET + CMDQ_THR_SHIFT * tid;
> + pc = readl(pc_va);
> + writel(pc, pc_va);
> +}
> +
> +static int cmdq_task_insert_into_thread(struct cmdq_task *task,
> + int tid, int loop)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + struct cmdq_task *prev_task;
> + int index, prev;
> +
> + /* find previous task and then link this task behind it */
> +
> + index = thread->next_cookie % CMDQ_MAX_TASK_IN_THREAD;
> + prev = (index + CMDQ_MAX_TASK_IN_THREAD - 1) % CMDQ_MAX_TASK_IN_THREAD;
> +
> + prev_task = thread->cur_task[prev];
> +
> + /* maybe the job is killed, search a new one */
> + for (; !prev_task && loop > 1; loop--) {
> + dev_err(dev,
> + "prev_task is NULL, prev:%d, loop:%d, index:%d\n",
> + prev, loop, index);
> +
> + prev--;
> + if (prev < 0)
> + prev = CMDQ_MAX_TASK_IN_THREAD - 1;
> +
> + prev_task = thread->cur_task[prev];
> + }
> +
> + if (!prev_task) {
> + dev_err(dev,
> + "invalid prev_task index:%d, loop:%d\n",
> + index, loop);
> + return -EFAULT;
> + }
> +
> + /* insert this task */
> + thread->cur_task[index] = task;
> + /* let previous task jump to this new task */
> + prev_task->va_base[prev_task->num_cmd - 1] = CMDQ_JUMP_BY_PA;
> + prev_task->va_base[prev_task->num_cmd - 2] = task->mva_base;
> +
> + /* re-fetch command buffer again. */
> + cmdq_core_invalidate_hw_fetched_buffer(cqctx, tid);
> +
> + return 0;
> +}
> +
> +static bool cmdq_command_is_wfe(u32 *cmd)
> +{
> + u32 wfe_option = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT | CMDQ_WFE_WAIT_VALUE;
> + u32 wfe_op = CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT;
> +
> + return (cmd[0] == wfe_option && (cmd[1] & CMDQ_OP_CODE_MASK) == wfe_op);
> +}
> +
> +/* we assume tasks in the same display thread are waiting the same event. */
> +static void cmdq_task_remove_wfe(struct cmdq_task *task)
> +{
> + u32 *base = task->va_base;
> + int i;
> +
> + /*
> + * Replace all WFE commands in the task command queue and
> + * replace them with JUMP_PASS.
> + */
> + for (i = 0; i < task->num_cmd; i += 2) {
> + if (cmdq_command_is_wfe(&base[i])) {
> + base[i] = CMDQ_JUMP_PASS;
> + base[i + 1] = CMDQ_JUMP_BY_OFFSET;
> + }
> + }
> +}
> +
> +static bool cmdq_thread_is_in_wfe(struct cmdq *cqctx, int tid)
> +{
> + return readl(cqctx->base + CMDQ_THR_WAIT_TOKEN_OFFSET +
> + CMDQ_THR_SHIFT * tid) & CMDQ_THR_IS_WAITING;
> +}
> +
> +static void cmdq_thread_wait_end(struct cmdq *cqctx, int tid,
> + unsigned long end_pa)
> +{
> + void __iomem *gce_base = cqctx->base;
> + unsigned long curr_pa;
> +
> + if (readl_poll_timeout_atomic(
> + gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid,
> + curr_pa, curr_pa == end_pa, 1, 20)) {
> + dev_err(cqctx->dev, "GCE thread(%d) cannot run to end.\n", tid);
> + }
> +}
> +
> +static int cmdq_task_exec_async_impl(struct cmdq_task *task, int tid)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + int status;
> + struct cmdq_thread *thread;
> + unsigned long flags;
> + int loop;
> + int minimum;
> + int cookie;
> +
> + status = 0;
> + thread = &cqctx->thread[tid];
> +
> + spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> + /* update task's thread info */
> + task->thread = tid;
> + task->irq_flag = 0;
> + task->task_state = TASK_STATE_BUSY;
> +
> + /* case 1. first task for this thread */
> + if (thread->task_count <= 0) {
> + if (cmdq_thread_reset(cqctx, tid) < 0) {
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> + return -EFAULT;
> + }
> +
> + writel(CMDQ_THR_NO_TIMEOUT,
> + gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + writel(task->mva_base,
> + gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + writel(task->mva_base + task->command_size,
> + gce_base + CMDQ_THR_END_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + writel(CMDQ_THR_PRIORITY,
> + gce_base + CMDQ_THR_CFG_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + writel(CMDQ_THR_IRQ_EN,
> + gce_base + CMDQ_THR_IRQ_ENABLE_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + minimum = cmdq_thread_get_cookie(cqctx, tid);
> + cmdq_thread_insert_task_by_cookie(
> + thread, task, (minimum + 1));
> +
> + /* enable HW thread */
> + writel(CMDQ_THR_ENABLED,
> + gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + } else {
> + unsigned long curr_pa, end_pa;
> +
> + status = cmdq_thread_suspend(cqctx, tid);
> + if (status < 0) {
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> + return status;
> + }
> +
> + writel(CMDQ_THR_NO_TIMEOUT,
> + gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + cookie = thread->next_cookie;
> +
> + curr_pa = (unsigned long)readl(gce_base +
> + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + end_pa = (unsigned long)readl(gce_base +
> + CMDQ_THR_END_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + /*
> + * case 2. If already exited WFE, wait for current task to end
> + * and then jump directly to new task.
> + */
> + if (!cmdq_thread_is_in_wfe(cqctx, tid)) {
> + cmdq_thread_resume(cqctx, tid);
> + cmdq_thread_wait_end(cqctx, tid, end_pa);
> + status = cmdq_thread_suspend(cqctx, tid);
> + if (status < 0) {
> + spin_unlock_irqrestore(&cqctx->exec_lock,
> + flags);
> + return status;
> + }
> + /* set to task directly */
> + writel(task->mva_base,
> + gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + writel(task->mva_base + task->command_size,
> + gce_base + CMDQ_THR_END_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
> + thread->task_count++;
> +
> + /*
> + * case 3. If thread is still in WFE from previous task, clear
> + * WFE in new task and append to thread.
> + */
> + } else {
> + /* Current task that shuld be processed */
> + minimum = cmdq_thread_get_cookie(cqctx, tid) + 1;
> + if (minimum > CMDQ_MAX_COOKIE_VALUE)
> + minimum = 0;
> +
> + /* Calculate loop count to adjust the tasks' order */
> + if (minimum <= cookie)
> + loop = cookie - minimum;
> + else
> + /* Counter wrapped */
> + loop = (CMDQ_MAX_COOKIE_VALUE - minimum + 1) +
> + cookie;
> +
> + if (loop < 0) {
> + dev_err(dev, "reorder fail:\n");
> + dev_err(dev, " task count=%d\n", loop);
> + dev_err(dev, " thread=%d\n", tid);
> + dev_err(dev, " next cookie=%d\n",
> + thread->next_cookie);
> + dev_err(dev, " (HW) next cookie=%d\n",
> + minimum);
> + dev_err(dev, " task=0x%p\n", task);
> +
> + spin_unlock_irqrestore(&cqctx->exec_lock,
> + flags);
> + return -EFAULT;
> + }
> +
> + if (loop > CMDQ_MAX_TASK_IN_THREAD)
> + loop %= CMDQ_MAX_TASK_IN_THREAD;
> +
> + status = cmdq_task_insert_into_thread(task, tid, loop);
> + if (status < 0) {
> + spin_unlock_irqrestore(
> + &cqctx->exec_lock, flags);
> + dev_err(dev,
> + "invalid task state for reorder.\n");
> + return status;
> + }
> +
> + cmdq_task_remove_wfe(task);
> +
> + smp_mb(); /* modify jump before enable thread */
> +
> + writel(task->mva_base + task->command_size,
> + gce_base + CMDQ_THR_END_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + thread->task_count++;
> + }
> +
> + thread->next_cookie += 1;
> + if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
> + thread->next_cookie = 0;
> +
> + /* resume HW thread */
> + cmdq_thread_resume(cqctx, tid);
> + }
> +
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +
> + return status;
> +}
> +
> +static void cmdq_core_handle_error(struct cmdq *cqctx, int tid, int value)
> +{
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + struct cmdq_thread *thread;
> + struct cmdq_task *task;
> + int cookie;
> + int count;
> + int inner;
> + int status;
> + u32 curr_pa, end_pa;
> +
> + curr_pa = readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + end_pa = readl(gce_base + CMDQ_THR_END_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + dev_err(dev, "IRQ: error thread=%d, irq_flag=0x%x\n", tid, value);
> + dev_err(dev, "IRQ: Thread PC: 0x%08x, End PC:0x%08x\n",
> + curr_pa, end_pa);
> +
> + thread = &cqctx->thread[tid];
> +
> + cookie = cmdq_thread_get_cookie(cqctx, tid);
> +
> + /*
> + * we assume error happens BEFORE EOC
> + * because it wouldn't be error if this interrupt is issue by EOC.
> + * so we should inc by 1 to locate "current" task
> + */
> + cookie++;
> +
> + /* set the issued task to error state */
> + if (thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD]) {
> + task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
> + task->irq_flag = value;
> + cmdq_thread_remove_task_by_index(
> + thread, cookie % CMDQ_MAX_TASK_IN_THREAD,
> + TASK_STATE_ERROR);
> + } else {
> + dev_err(dev,
> + "IRQ: can not find task in %s, pc:0x%08x, end_pc:0x%08x\n",
> + __func__, curr_pa, end_pa);
> + if (thread->task_count <= 0) {
> + /*
> + * suspend HW thread first,
> + * so that we work in a consistent state
> + * outer function should acquire spinlock:
> + * cqctx->exec_lock
> + */
> + status = cmdq_thread_suspend(cqctx, tid);
> + if (status < 0)
> + dev_err(dev, "IRQ: suspend HW thread failed!");
> +
> + cmdq_thread_disable(cqctx, tid);
> + dev_err(dev,
> + "IRQ: there is no task for thread (%d) %s\n",
> + tid, __func__);
> + }
> + }
> +
> + /* set the remain tasks to done state */
> + if (thread->wait_cookie <= cookie) {
> + count = cookie - thread->wait_cookie + 1;
> + } else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
> + thread->wait_cookie) {
> + count = 0;
> + } else {
> + /* counter wrapped */
> + count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
> + (cookie + 1);
> + dev_err(dev,
> + "IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
> + thread->wait_cookie, cookie, count);
> + }
> +
> + for (inner = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
> + count--, inner++) {
> + if (inner >= CMDQ_MAX_TASK_IN_THREAD)
> + inner = 0;
> +
> + if (thread->cur_task[inner]) {
> + task = thread->cur_task[inner];
> + task->irq_flag = 0; /* don't know irq flag */
> + /* still call isr_cb to prevent lock */
> + if (task->cb.isr_cb)
> + task->cb.isr_cb(task->cb.isr_data);
> + cmdq_thread_remove_task_by_index(
> + thread, inner, TASK_STATE_DONE);
> + }
> + }
> +
> + thread->wait_cookie = cookie + 1;
> + if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
> + thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
> + /* min cookie value is 0 */
> +
> + wake_up(&cqctx->wait_queue[tid]);
> +}
> +
> +static void cmdq_core_handle_done(struct cmdq *cqctx, int tid, int value)
> +{
> + struct device *dev = cqctx->dev;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + int cookie = cmdq_thread_get_cookie(cqctx, tid);
> + int count;
> + int i;
> + struct cmdq_task *task;
> +
> + if (thread->wait_cookie <= cookie) {
> + count = cookie - thread->wait_cookie + 1;
> + } else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
> + thread->wait_cookie) {
> + count = 0;
> + } else {
> + /* counter wrapped */
> + count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
> + (cookie + 1);
> + dev_err(dev,
> + "IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
> + thread->wait_cookie, cookie, count);
> + }
> +
> + for (i = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
> + count--, i++) {
> + if (i >= CMDQ_MAX_TASK_IN_THREAD)
> + i = 0;
> +
> + if (thread->cur_task[i]) {
> + task = thread->cur_task[i];
> + task->irq_flag = value;
> + if (task->cb.isr_cb)
> + task->cb.isr_cb(task->cb.isr_data);
> + cmdq_thread_remove_task_by_index(
> + thread, i, TASK_STATE_DONE);
> + }
> + }
> +
> + thread->wait_cookie = cookie + 1;
> + if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
> + thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
> + /* min cookie value is 0 */
> +
> + wake_up(&cqctx->wait_queue[tid]);
> +}
> +
> +static void cmdq_core_handle_irq(struct cmdq *cqctx, int tid)
> +{
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + unsigned long flags = 0L;
> + int value;
> + int enabled;
> + int cookie;
> +
> + /*
> + * normal execution, marks tasks done and remove from thread
> + * also, handle "loop CB fail" case
> + */
> + spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> + /*
> + * it is possible for another CPU core
> + * to run "release task" right before we acquire the spin lock
> + * and thus reset / disable this HW thread
> + * so we check both the IRQ flag and the enable bit of this thread
> + */
> + value = readl(gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + if (!(value & CMDQ_THR_IRQ_MASK)) {
> + dev_err(dev,
> + "IRQ: thread %d got interrupt but IRQ flag is 0x%08x\n",
> + tid, value);
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> + return;
> + }
> +
> + enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + if (!(enabled & CMDQ_THR_ENABLED)) {
> + dev_err(dev,
> + "IRQ: thread %d got interrupt already disabled 0x%08x\n",
> + tid, enabled);
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> + return;
> + }
> +
> + /* read HW cookie here for printing message */
> + cookie = cmdq_thread_get_cookie(cqctx, tid);
> +
> + /*
> + * Move the reset IRQ before read HW cookie
> + * to prevent race condition and save the cost of suspend
> + */
> + writel(~value,
> + gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + if (value & CMDQ_THR_IRQ_ERROR)
> + cmdq_core_handle_error(cqctx, tid, value);
> + else if (value & CMDQ_THR_IRQ_DONE)
> + cmdq_core_handle_done(cqctx, tid, value);
> +
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +}
> +
> +static int cmdq_task_exec_async(struct cmdq_task *task, int tid)
> +{
> + struct device *dev = task->cqctx->dev;
> + int status;
> +
> + status = cmdq_task_exec_async_impl(task, tid);
> + if (status >= 0)
> + return status;
> +
> + if ((task->task_state == TASK_STATE_KILLED) ||
> + (task->task_state == TASK_STATE_ERROR)) {
> + dev_err(dev, "cmdq_task_exec_async_impl fail\n");
> + return -EFAULT;
> + }
> +
> + return 0;
> +}
> +
> +static void cmdq_core_consume_waiting_list(struct work_struct *work)
> +{
> + struct list_head *p, *n = NULL;
> + bool thread_acquired;
> + ktime_t consume_time;
> + s64 waiting_time_ns;
> + bool need_log;
> + struct cmdq *cqctx;
> + struct device *dev;
> + u32 err_bits = 0;
> +
> + cqctx = container_of(work, struct cmdq,
> + task_consume_wait_queue_item);
> + dev = cqctx->dev;
> +
> + consume_time = ktime_get();
> +
> + mutex_lock(&cqctx->task_mutex);
> +
> + thread_acquired = false;
> +
> + /* scan and remove (if executed) waiting tasks */
> + list_for_each_safe(p, n, &cqctx->task_wait_list) {
> + struct cmdq_task *task;
> + struct cmdq_thread *thread;
> + int tid;
> + int status;
> +
> + task = list_entry(p, struct cmdq_task, list_entry);
> + tid = cmdq_eng_get_thread(task->engine_flag);
> +
> + waiting_time_ns = ktime_to_ns(
> + ktime_sub(consume_time, task->submit));
> + need_log = waiting_time_ns >= CMDQ_PREALARM_TIMEOUT_NS;
> +
> + /*
> + * Once waiting occur,
> + * skip following tasks to keep order of display tasks.
> + */
> + if (err_bits & BIT(tid))
> + continue;
> +
> + /* acquire HW thread */
> + thread = cmdq_core_acquire_thread(cqctx, tid);
> + if (!thread) {
> + /* have to wait, remain in wait list */
> + dev_warn(dev, "acquire thread(%d) fail, need to wait\n",
> + tid);
> + if (need_log) /* task wait too long */
> + dev_warn(dev, "waiting:%lldns, task:0x%p\n",
> + waiting_time_ns, task);
> + err_bits |= BIT(tid);
> + continue;
> + }
> +
> + /* some task is ready to run */
> + thread_acquired = true;
> +
> + /*
> + * start execution
> + * remove from wait list and put into active list
> + */
> + list_move_tail(&task->list_entry,
> + &cqctx->task_active_list);
> +
> + /* run task on thread */
> + status = cmdq_task_exec_async(task, tid);
> + if (status < 0) {
> + dev_err(dev, "%s fail, release task 0x%p\n",
> + __func__, task);
> + cmdq_task_remove_thread(task);
> + cmdq_task_release_unlocked(task);
> + task = NULL;
> + }
> + }
> +
> + if (thread_acquired) {
> + /*
> + * notify some task's sw thread to change their waiting state.
> + * (if they have already called cmdq_task_wait_and_release())
> + */
> + wake_up_all(&cqctx->thread_dispatch_queue);
> + }
> +
> + mutex_unlock(&cqctx->task_mutex);
> +}
> +
> +static int cmdq_core_submit_task_async(struct cmdq_command *cmd_desc,
> + struct cmdq_task **task_out,
> + struct cmdq_task_cb *cb)
> +{
> + struct cmdq *cqctx = cmd_desc->cqctx;
> +
> + /* creates a new task and put into tail of waiting list */
> + *task_out = cmdq_core_acquire_task(cmd_desc, cb);
> +
> + if (!(*task_out))
> + return -EFAULT;
> +
> + /*
> + * Consume the waiting list.
> + * This may or may not execute the task, depending on available threads.
> + */
> + cmdq_core_consume_waiting_list(&cqctx->task_consume_wait_queue_item);
> +
> + return 0;
> +}
> +
> +static int cmdq_core_release_task(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + int tid = task->thread;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + unsigned long flags;
> + int status;
> +
> + if (tid != CMDQ_INVALID_THREAD && thread) {
> + /* this task is being executed (or queueed) on a hw thread */
> +
> + /* get sw lock first to ensure atomic access hw */
> + spin_lock_irqsave(&cqctx->exec_lock, flags);
> + smp_mb(); /* make sure atomic access hw */
> +
> + status = cmdq_thread_force_remove_task(task, tid);
> + if (thread->task_count > 0)
> + cmdq_thread_resume(cqctx, tid);
> +
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> + wake_up(&cqctx->wait_queue[tid]);
> + }
> +
> + cmdq_task_remove_thread(task);
> + cmdq_task_release_internal(task);
> + return 0;
> +}
> +
> +struct cmdq_task_error_report {
> + bool throw_err;
> + const char *module;
> + u32 inst_a;
> + u32 inst_b;
> + u32 irq_flag;
> +};
> +
> +static int cmdq_task_handle_error_result(
> + struct cmdq_task *task, int tid, int wait_q,
> + struct cmdq_task_error_report *error_report)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + void __iomem *gce_base = cqctx->base;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + int status = 0;
> + int i;
> + bool is_err = false;
> + struct cmdq_task *next_task;
> + struct cmdq_task *prev_task;
> + int cookie;
> + unsigned long thread_pc;
> +
> + dev_err(dev,
> + "task(0x%p) state is not TASK_STATE_DONE, but %d.\n",
> + task, task->task_state);
> +
> + /*
> + * Oops, that task is not done.
> + * We have several possible error cases:
> + * 1. task still running (hang / timeout)
> + * 2. IRQ pending (done or error/timeout IRQ)
> + * 3. task's SW thread has been signaled (e.g. SIGKILL)
> + */
> +
> + /*
> + * suspend HW thread first,
> + * so that we work in a consistent state
> + */
> + status = cmdq_thread_suspend(cqctx, tid);
> + if (status < 0)
> + error_report->throw_err = true;
> +
> + /* The cookie of the task currently being processed */
> + cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
> + thread_pc = (unsigned long)readl(gce_base +
> + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + /* process any pending IRQ */
> + error_report->irq_flag = readl(
> + gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + if (error_report->irq_flag & CMDQ_THR_IRQ_ERROR)
> + cmdq_core_handle_error(cqctx, tid, error_report->irq_flag);
> + else if (error_report->irq_flag & CMDQ_THR_IRQ_DONE)
> + cmdq_core_handle_done(cqctx, tid, error_report->irq_flag);
> +
> + writel(~error_report->irq_flag,
> + gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> +
> + /* check if this task has finished after handling pending IRQ */
> + if (task->task_state == TASK_STATE_DONE)
> + return 0;
> +
> + /* Then decide we are SW timeout or SIGNALed (not an error) */
> + if (!wait_q) {
> + /* SW timeout and no IRQ received */
> + is_err = true;
> + dev_err(dev, "SW timeout of task 0x%p on tid %d\n",
> + task, tid);
> + error_report->throw_err = true;
> + cmdq_core_parse_error(task, tid,
> + &error_report->module,
> + &error_report->irq_flag,
> + &error_report->inst_a,
> + &error_report->inst_b);
> + status = -ETIMEDOUT;
> + } else if (wait_q < 0) {
> + /*
> + * Task is killed.
> + * Not an error, but still need to remove.
> + */
> + is_err = false;
> +
> + if (wait_q == -ERESTARTSYS)
> + dev_err(dev,
> + "Task 0x%p KILLED by wait_q = -ERESTARTSYS\n",
> + task);
> + else if (wait_q == -EINTR)
> + dev_err(dev,
> + "Task 0x%p KILLED by wait_q = -EINTR\n",
> + task);
> + else
> + dev_err(dev,
> + "Task 0x%p KILLED by wait_q = %d\n",
> + task, wait_q);
> +
> + status = wait_q;
> + }
> +
> + if (task->task_state == TASK_STATE_BUSY) {
> + /*
> + * if task_state is BUSY,
> + * this means we did not reach EOC,
> + * did not have error IRQ.
> + * - remove the task from thread.cur_task[]
> + * - and decrease thread.task_count
> + * NOTE: after this,
> + * the cur_task will not contain link to task anymore.
> + * and task should become TASK_STATE_ERROR
> + */
> +
> + /* we find our place in thread->cur_task[]. */
> + for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> + if (thread->cur_task[i] == task) {
> + /* update task_count and cur_task[] */
> + cmdq_thread_remove_task_by_index(
> + thread, i, is_err ?
> + TASK_STATE_ERROR :
> + TASK_STATE_KILLED);
> + break;
> + }
> + }
> + }
> +
> + next_task = NULL;
> +
> + /* find task's jump destination or no next task*/
> + if (task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_PA)
> + next_task = cmdq_thread_search_task_by_pc(
> + thread,
> + task->va_base[task->num_cmd - 2]);
> +
> + /*
> + * Then, we try remove task from the chain of thread->cur_task.
> + * . if HW PC falls in task range
> + * . HW EXEC_CNT += 1
> + * . thread.wait_cookie += 1
> + * . set HW PC to next task head
> + * . if not, find previous task
> + * (whose jump address is task->mva_base)
> + * . check if HW PC points is not at the EOC/JUMP end
> + * . change jump to fake EOC(no IRQ)
> + * . insert jump to next task head and increase cmd buffer size
> + * . if there is no next task, set HW End Address
> + */
> + if (task->num_cmd && thread_pc >= task->mva_base &&
> + thread_pc <= (task->mva_base + task->command_size)) {
> + if (next_task) {
> + /* cookie already +1 */
> + writel(cookie,
> + gce_base + CMDQ_THR_EXEC_CNT_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + thread->wait_cookie = cookie + 1;
> + writel(next_task->mva_base,
> + gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> + CMDQ_THR_SHIFT * tid);
> + }
> + } else {
> + prev_task = NULL;
> + for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> + u32 *prev_va, *curr_va;
> + u32 prev_num, curr_num;
> +
> + prev_task = thread->cur_task[i];
> + if (!prev_task)
> + continue;
> +
> + prev_va = prev_task->va_base;
> + prev_num = prev_task->num_cmd;
> + if (!prev_num)
> + continue;
> +
> + curr_va = task->va_base;
> + curr_num = task->num_cmd;
> +
> + /* find which task JUMP into task */
> + if (prev_va[prev_num - 2] == task->mva_base &&
> + prev_va[prev_num - 1] == CMDQ_JUMP_BY_PA) {
> + /* Copy Jump instruction */
> + prev_va[prev_num - 2] =
> + curr_va[curr_num - 2];
> + prev_va[prev_num - 1] =
> + curr_va[curr_num - 1];
> +
> + if (next_task)
> + cmdq_thread_reorder_task_array(
> + thread, i);
> +
> + /*
> + * Give up fetched command,
> + * invoke CMDQ HW to re-fetch command.
> + */
> + cmdq_core_invalidate_hw_fetched_buffer(
> + cqctx, tid);
> +
> + break;
> + }
> + }
> + }
> +
> + return status;
> +}
> +
> +static int cmdq_task_wait_result(struct cmdq_task *task, int tid, int wait_q)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct cmdq_thread *thread = &cqctx->thread[tid];
> + int status = 0;
> + unsigned long flags;
> + struct cmdq_task_error_report error_report = { 0 };
> +
> + /*
> + * Note that although we disable IRQ, HW continues to execute
> + * so it's possible to have pending IRQ
> + */
> + spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> + if (task->task_state != TASK_STATE_DONE)
> + status = cmdq_task_handle_error_result(
> + task, tid, wait_q, &error_report);
> +
> + if (thread->task_count <= 0)
> + cmdq_thread_disable(cqctx, tid);
> + else
> + cmdq_thread_resume(cqctx, tid);
> +
> + spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +
> + if (error_report.throw_err) {
> + u32 op = error_report.inst_a >> CMDQ_OP_CODE_SHIFT;
> +
> + switch (op) {
> + case CMDQ_CODE_WFE:
> + dev_err(cqctx->dev,
> + "%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:WAIT EVENT:%s\n",
> + error_report.module, error_report.irq_flag,
> + error_report.inst_a, error_report.inst_b,
> + cmdq_event_get_name(error_report.inst_a &
> + CMDQ_ARG_A_MASK));
> + break;
> + default:
> + dev_err(cqctx->dev,
> + "%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:%s\n",
> + error_report.module, error_report.irq_flag,
> + error_report.inst_a, error_report.inst_b,
> + cmdq_core_parse_op(op));
> + break;
> + }
> + }
> +
> + return status;
> +}
> +
> +static int cmdq_task_wait_done(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> + struct device *dev = cqctx->dev;
> + int wait_q;
> + int tid;
> + unsigned long timeout = msecs_to_jiffies(
> + CMDQ_ACQUIRE_THREAD_TIMEOUT_MS);
> +
> + /*
> + * wait for acquire thread
> + * (this is done by cmdq_core_consume_waiting_list);
> + */
> + wait_q = wait_event_timeout(
> + cqctx->thread_dispatch_queue,
> + (task->thread != CMDQ_INVALID_THREAD), timeout);
> +
> + if (!wait_q) {
> + mutex_lock(&cqctx->task_mutex);
> +
> + /*
> + * it's possible that the task was just consumed now.
> + * so check again.
> + */
> + if (task->thread == CMDQ_INVALID_THREAD) {
> + /*
> + * Task may have released,
> + * or starved to death.
> + */
> + dev_err(dev,
> + "task(0x%p) timeout with invalid thread\n",
> + task);
> +
> + /*
> + * remove from waiting list,
> + * so that it won't be consumed in the future
> + */
> + list_del_init(&task->list_entry);
> +
> + mutex_unlock(&cqctx->task_mutex);
> + return -EINVAL;
> + }
> +
> + /* valid thread, so we keep going */
> + mutex_unlock(&cqctx->task_mutex);
> + }
> +
> + tid = task->thread;
> + if (tid < 0 || tid >= CMDQ_MAX_THREAD_COUNT) {
> + dev_err(dev, "invalid thread %d in %s\n", tid, __func__);
> + return -EINVAL;
> + }
> +
> + /* start to wait */
> + wait_q = wait_event_timeout(task->cqctx->wait_queue[tid],
> + (task->task_state != TASK_STATE_BUSY &&
> + task->task_state != TASK_STATE_WAITING),
> + msecs_to_jiffies(CMDQ_DEFAULT_TIMEOUT_MS));
> + if (!wait_q)
> + dev_dbg(dev, "timeout!\n");
> +
> + /* wake up and continue */
> + return cmdq_task_wait_result(task, tid, wait_q);
> +}
> +
> +static int cmdq_task_wait_and_release(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx;
> + int status;
> +
> + if (!task) {
> + pr_err("%s err ptr=0x%p\n", __func__, task);
> + return -EFAULT;
> + }
> +
> + if (task->task_state == TASK_STATE_IDLE) {
> + pr_err("%s task=0x%p is IDLE\n", __func__, task);
> + return -EFAULT;
> + }
> +
> + cqctx = task->cqctx;
> +
> + /* wait for task finish */
> + status = cmdq_task_wait_done(task);
> +
> + /* release */
> + cmdq_task_remove_thread(task);
> + cmdq_task_release_internal(task);
> +
> + return status;
> +}
> +
> +static void cmdq_core_auto_release_work(struct work_struct *work_item)
> +{
> + struct cmdq_task *task;
> + int status;
> + struct cmdq_task_cb cb;
> +
> + task = container_of(work_item, struct cmdq_task, auto_release_work);
> + cb = task->cb;
> + status = cmdq_task_wait_and_release(task);
> +
> + /* isr fail, so call isr_cb here to prevent lock */
> + if (status && cb.isr_cb)
> + cb.isr_cb(cb.isr_data);
> +
> + if (cb.done_cb)
> + cb.done_cb(cb.done_data);
> +}
> +
> +static int cmdq_core_auto_release_task(struct cmdq_task *task)
> +{
> + struct cmdq *cqctx = task->cqctx;
> +
> + /*
> + * the work item is embeded in task already
> + * but we need to initialized it
> + */
> + INIT_WORK(&task->auto_release_work, cmdq_core_auto_release_work);
> + queue_work(cqctx->task_auto_release_wq, &task->auto_release_work);
> + return 0;
> +}
> +
> +static int cmdq_core_submit_task(struct cmdq_command *cmd_desc)
> +{
> + struct device *dev = cmd_desc->cqctx->dev;
> + int status;
> + struct cmdq_task *task;
> +
> + status = cmdq_core_submit_task_async(cmd_desc, &task, NULL);
> + if (status < 0) {
> + dev_err(dev, "cmdq_core_submit_task_async failed=%d\n", status);
> + return status;
> + }
> +
> + status = cmdq_task_wait_and_release(task);
> + if (status < 0)
> + dev_err(dev, "task(0x%p) wait fail\n", task);
> +
> + return status;
> +}
> +
> +static void cmdq_core_deinitialize(struct platform_device *pdev)
> +{
> + struct cmdq *cqctx = platform_get_drvdata(pdev);
> + int i;
> + struct list_head *lists[] = {
> + &cqctx->task_free_list,
> + &cqctx->task_active_list,
> + &cqctx->task_wait_list
> + };
> +
> + /*
> + * Directly destroy the auto release WQ
> + * since we're going to release tasks anyway.
> + */
> + destroy_workqueue(cqctx->task_auto_release_wq);
> + cqctx->task_auto_release_wq = NULL;
> +
> + destroy_workqueue(cqctx->task_consume_wq);
> + cqctx->task_consume_wq = NULL;
> +
> + /* release all tasks in both list */
> + for (i = 0; i < ARRAY_SIZE(lists); i++) {
> + struct cmdq_task *task, *tmp;
> +
> + list_for_each_entry_safe(task, tmp, lists[i], list_entry) {
> + cmdq_task_free_command_buffer(task);
> + kmem_cache_free(cqctx->task_cache, task);
> + list_del(&task->list_entry);
> + }
> + }
> +
> + kmem_cache_destroy(cqctx->task_cache);
> + cqctx->task_cache = NULL;
> +
> + /* release command buffer pool */
> + cmdq_cmd_buf_pool_uninit(cqctx);
> +}
> +
> +static irqreturn_t cmdq_irq_handler(int irq, void *dev)
> +{
> + struct cmdq *cqctx = dev;
> + int i;
> + u32 irq_status;
> + bool handled = false;
> +
> + irq_status = readl(cqctx->base + CMDQ_CURR_IRQ_STATUS_OFFSET);
> + irq_status &= CMDQ_IRQ_MASK;
> + for (i = 0;
> + irq_status != CMDQ_IRQ_MASK && i < CMDQ_MAX_THREAD_COUNT;
> + i++) {
> + /* STATUS bit set to 0 means IRQ asserted */
> + if (irq_status & BIT(i))
> + continue;
> +
> + /*
> + * We mark irq_status to 1 to denote finished
> + * processing, and we can early-exit if no more
> + * threads being asserted.
> + */
> + irq_status |= BIT(i);
> +
> + cmdq_core_handle_irq(cqctx, i);
> + handled = true;
> + }
> +
> + if (!handled)
> + return IRQ_NONE;
> +
> + queue_work(cqctx->task_consume_wq,
> + &cqctx->task_consume_wait_queue_item);
> + return IRQ_HANDLED;
> +}
> +
> +static int cmdq_core_initialize(struct platform_device *pdev,
> + struct cmdq **cqctx)
> +{
> + struct cmdq *lcqctx; /* local cmdq context */
> + int i;
> + int ret = 0;
> +
> + lcqctx = devm_kzalloc(&pdev->dev, sizeof(*lcqctx), GFP_KERNEL);
> +
> + /* save dev */
> + lcqctx->dev = &pdev->dev;
> +
> + /* initial cmdq device related data */
> + ret = cmdq_dev_init(pdev, lcqctx);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to init cmdq device\n");
> + goto fail_dev;
> + }
> +
> + /* initial mutex, spinlock */
> + mutex_init(&lcqctx->task_mutex);
> + mutex_init(&lcqctx->clock_mutex);
> + spin_lock_init(&lcqctx->thread_lock);
> + spin_lock_init(&lcqctx->exec_lock);
> +
> + /* initial wait queue for notification */
> + for (i = 0; i < ARRAY_SIZE(lcqctx->wait_queue); i++)
> + init_waitqueue_head(&lcqctx->wait_queue[i]);
> + init_waitqueue_head(&lcqctx->thread_dispatch_queue);
> +
> + /* create task pool */
> + lcqctx->task_cache = kmem_cache_create(
> + CMDQ_DRIVER_DEVICE_NAME "_task",
> + sizeof(struct cmdq_task),
> + __alignof__(struct cmdq_task),
> + SLAB_POISON | SLAB_HWCACHE_ALIGN | SLAB_RED_ZONE,
> + &cmdq_task_ctor);
> +
> + /* initialize task lists */
> + INIT_LIST_HEAD(&lcqctx->task_free_list);
> + INIT_LIST_HEAD(&lcqctx->task_active_list);
> + INIT_LIST_HEAD(&lcqctx->task_wait_list);
> + INIT_WORK(&lcqctx->task_consume_wait_queue_item,
> + cmdq_core_consume_waiting_list);
> +
> + /* initialize command buffer pool */
> + ret = cmdq_cmd_buf_pool_init(lcqctx);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to init command buffer pool\n");
> + goto fail_cmd_buf_pool;
> + }
> +
> + lcqctx->task_auto_release_wq = alloc_ordered_workqueue(
> + "%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_auto_release");
> + lcqctx->task_consume_wq = alloc_ordered_workqueue(
> + "%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_task");
> +
> + *cqctx = lcqctx;
> + return ret;
> +
> +fail_cmd_buf_pool:
> + destroy_workqueue(lcqctx->task_auto_release_wq);
> + destroy_workqueue(lcqctx->task_consume_wq);
> + kmem_cache_destroy(lcqctx->task_cache);
> +
> +fail_dev:
> + return ret;
> +}
> +
> +static int cmdq_rec_realloc_cmd_buffer(struct cmdq_rec *handle, size_t size)
> +{
> + void *new_buf;
> +
> + new_buf = krealloc(handle->buf_ptr, size, GFP_KERNEL | __GFP_ZERO);
> + if (!new_buf)
> + return -ENOMEM;
> + handle->buf_ptr = new_buf;
> + handle->buf_size = size;
> + return 0;
> +}
> +
> +static int cmdq_rec_stop_running_task(struct cmdq_rec *handle)
> +{
> + int status;
> +
> + status = cmdq_core_release_task(handle->running_task_ptr);
> + handle->running_task_ptr = NULL;
> + return status;
> +}
> +
> +int cmdq_rec_create(struct device *dev, u64 engine_flag,
> + struct cmdq_rec **handle_ptr)
> +{
> + struct cmdq *cqctx;
> + struct cmdq_rec *handle;
> + int ret;
> +
> + cqctx = dev_get_drvdata(dev);
> + if (!cqctx) {
> + dev_err(dev, "cmdq context is NULL\n");
> + return -EINVAL;
> + }
> +
> + handle = kzalloc(sizeof(*handle), GFP_KERNEL);
> + if (!handle)
> + return -ENOMEM;
> +
> + handle->cqctx = dev_get_drvdata(dev);
> + handle->engine_flag = engine_flag;
> +
> + ret = cmdq_rec_realloc_cmd_buffer(handle, CMDQ_INITIAL_CMD_BLOCK_SIZE);
> + if (ret) {
> + kfree(handle);
> + return ret;
> + }
> +
> + *handle_ptr = handle;
> +
> + return 0;
> +}
> +EXPORT_SYMBOL(cmdq_rec_create);
> +
> +static int cmdq_rec_append_command(struct cmdq_rec *handle,
> + enum cmdq_code code,
> + u32 arg_a, u32 arg_b)
> +{
> + struct cmdq *cqctx;
> + struct device *dev;
> + int subsys;
> + u32 *cmd_ptr;
> + int ret;
> +
> + cqctx = handle->cqctx;
> + dev = cqctx->dev;
> + cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
> +
> + if (handle->finalized) {
> + dev_err(dev,
> + "already finalized record(cannot add more command)");
> + dev_err(dev, "handle=0x%p, tid=%d\n", handle, current->pid);
> + return -EBUSY;
> + }
> +
> + /* check if we have sufficient buffer size */
> + if (unlikely(handle->block_size + CMDQ_INST_SIZE > handle->buf_size)) {
> + ret = cmdq_rec_realloc_cmd_buffer(handle, handle->buf_size * 2);
> + if (ret)
> + return ret;
> + }
> +
> + /*
> + * we must re-calculate current PC
> + * because we may already insert MARKER inst.
> + */
> + cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
> +
> + switch (code) {
> + case CMDQ_CODE_MOVE:
> + cmd_ptr[0] = arg_b;
> + cmd_ptr[1] = (CMDQ_CODE_MOVE << CMDQ_OP_CODE_SHIFT) |
> + (arg_a & CMDQ_ARG_A_MASK);
> + break;
> + case CMDQ_CODE_WRITE:
> + subsys = cmdq_subsys_from_phys_addr(cqctx, arg_a);
> + if (subsys < 0) {
> + dev_err(dev,
> + "unsupported memory base address 0x%08x\n",
> + arg_a);
> + return -EFAULT;
> + }
> +
> + cmd_ptr[0] = arg_b;
> + cmd_ptr[1] = (CMDQ_CODE_WRITE << CMDQ_OP_CODE_SHIFT) |
> + (arg_a & CMDQ_ARG_A_WRITE_MASK) |
> + ((subsys & CMDQ_SUBSYS_MASK) << CMDQ_SUBSYS_SHIFT);
> + break;
> + case CMDQ_CODE_JUMP:
> + cmd_ptr[0] = arg_b;
> + cmd_ptr[1] = (CMDQ_CODE_JUMP << CMDQ_OP_CODE_SHIFT) |
> + (arg_a & CMDQ_ARG_A_MASK);
> + break;
> + case CMDQ_CODE_WFE:
> + /*
> + * bit 0-11: wait_value, 1
> + * bit 15: to_wait, true
> + * bit 16-27: update_value, 0
> + * bit 31: to_update, true
> + */
> + cmd_ptr[0] = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT |
> + CMDQ_WFE_WAIT_VALUE;
> + cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
> + break;
> + case CMDQ_CODE_CLEAR_EVENT:
> + /*
> + * bit 0-11: wait_value, 0
> + * bit 15: to_wait, false
> + * bit 16-27: update_value, 0
> + * bit 31: to_update, true
> + */
> + cmd_ptr[0] = CMDQ_WFE_UPDATE;
> + cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
> + break;
> + case CMDQ_CODE_EOC:
> + cmd_ptr[0] = arg_b;
> + cmd_ptr[1] = (CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT) |
> + (arg_a & CMDQ_ARG_A_MASK);
> + break;
> + default:
> + return -EFAULT;
> + }
> +
> + handle->block_size += CMDQ_INST_SIZE;
> +
> + return 0;
> +}
> +
> +int cmdq_rec_reset(struct cmdq_rec *handle)
> +{
> + if (handle->running_task_ptr)
> + cmdq_rec_stop_running_task(handle);
> +
> + handle->block_size = 0;
> + handle->finalized = false;
> +
> + return 0;
> +}
> +EXPORT_SYMBOL(cmdq_rec_reset);
> +
> +int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr)
> +{
> + return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
> +}
> +EXPORT_SYMBOL(cmdq_rec_write);
> +
> +int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
> + u32 addr, u32 mask)
> +{
> + int ret;
> +
> + if (mask != 0xffffffff) {
> + ret = cmdq_rec_append_command(handle, CMDQ_CODE_MOVE, 0, ~mask);
> + if (ret)
> + return ret;
> +
> + addr = addr | CMDQ_ENABLE_MASK;
> + }
> +
> + return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
> +}
> +EXPORT_SYMBOL(cmdq_rec_write_mask);
> +
> +int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event)
> +{
> + if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
> + event < 0)
> + return -EINVAL;
> +
> + return cmdq_rec_append_command(handle, CMDQ_CODE_WFE, event, 0);
> +}
> +EXPORT_SYMBOL(cmdq_rec_wait);
> +
> +int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event)
> +{
> + if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
> + event < 0)
> + return -EINVAL;
> +
> + return cmdq_rec_append_command(handle, CMDQ_CODE_CLEAR_EVENT, event, 0);
> +}
> +EXPORT_SYMBOL(cmdq_rec_clear_event);
> +
> +static int cmdq_rec_finalize_command(struct cmdq_rec *handle)
> +{
> + int status;
> + struct device *dev;
> + u32 arg_b;
> +
> + dev = handle->cqctx->dev;
> +
> + if (!handle->finalized) {
> + /* insert EOC and generate IRQ for each command iteration */
> + arg_b = CMDQ_EOC_IRQ_EN;
> + status = cmdq_rec_append_command(handle, CMDQ_CODE_EOC,
> + 0, arg_b);
> + if (status)
> + return status;
> +
> + /* JUMP to begin */
> + status = cmdq_rec_append_command(handle, CMDQ_CODE_JUMP, 0, 8);
> + if (status)
> + return status;
> +
> + handle->finalized = true;
> + }
> +
> + return 0;
> +}
> +
> +static int cmdq_rec_fill_cmd_desc(struct cmdq_rec *handle,
> + struct cmdq_command *desc)
> +{
> + int ret;
> +
> + ret = cmdq_rec_finalize_command(handle);
> + if (ret)
> + return ret;
> +
> + desc->cqctx = handle->cqctx;
> + desc->engine_flag = handle->engine_flag;
> + desc->va_base = handle->buf_ptr;
> + desc->block_size = handle->block_size;
> +
> + return ret;
> +}
> +
> +int cmdq_rec_flush(struct cmdq_rec *handle)
> +{
> + int ret;
> + struct cmdq_command desc;
> +
> + ret = cmdq_rec_fill_cmd_desc(handle, &desc);
> + if (ret)
> + return ret;
> +
> + return cmdq_core_submit_task(&desc);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush);
> +
> +static int cmdq_rec_flush_async_cb(struct cmdq_rec *handle,
> + cmdq_async_flush_cb isr_cb,
> + void *isr_data,
> + cmdq_async_flush_cb done_cb,
> + void *done_data)
> +{
> + int ret;
> + struct cmdq_command desc;
> + struct cmdq_task *task;
> + struct cmdq_task_cb cb;
> +
> + ret = cmdq_rec_fill_cmd_desc(handle, &desc);
> + if (ret)
> + return ret;
> +
> + cb.isr_cb = isr_cb;
> + cb.isr_data = isr_data;
> + cb.done_cb = done_cb;
> + cb.done_data = done_data;
> +
> + ret = cmdq_core_submit_task_async(&desc, &task, &cb);
> + if (ret)
> + return ret;
> +
> + ret = cmdq_core_auto_release_task(task);
> +
> + return ret;
> +}
> +
> +int cmdq_rec_flush_async(struct cmdq_rec *handle)
> +{
> + return cmdq_rec_flush_async_cb(handle, NULL, NULL, NULL, NULL);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush_async);
> +
> +int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
> + cmdq_async_flush_cb isr_cb,
> + void *isr_data,
> + cmdq_async_flush_cb done_cb,
> + void *done_data)
> +{
> + return cmdq_rec_flush_async_cb(handle, isr_cb, isr_data,
> + done_cb, done_data);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush_async_callback);
> +
> +void cmdq_rec_destroy(struct cmdq_rec *handle)
> +{
> + if (handle->running_task_ptr)
> + cmdq_rec_stop_running_task(handle);
> +
> + /* free command buffer */
> + kfree(handle->buf_ptr);
> + handle->buf_ptr = NULL;
> +
> + /* free command handle */
> + kfree(handle);
> +}
> +EXPORT_SYMBOL(cmdq_rec_destroy);
> +
> +static int cmdq_probe(struct platform_device *pdev)
> +{
> + struct cmdq *cqctx;
> + int ret;
> +
> + /* init cmdq context, and save it */
> + ret = cmdq_core_initialize(pdev, &cqctx);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to init cmdq context\n");
> + return ret;
> + }
> + platform_set_drvdata(pdev, cqctx);
> +
> + ret = devm_request_irq(&pdev->dev, cqctx->irq, cmdq_irq_handler,
> + IRQF_TRIGGER_LOW | IRQF_SHARED,
> + CMDQ_DRIVER_DEVICE_NAME, cqctx);
> + if (ret) {
> + dev_err(&pdev->dev, "failed to register ISR (%d)\n", ret);
> + goto fail;
> + }
> +
> + cqctx->clock = devm_clk_get(&pdev->dev, CMDQ_CLK_NAME);
> + if (IS_ERR(cqctx->clock)) {
> + dev_err(&pdev->dev, "failed to get clk:%s\n", CMDQ_CLK_NAME);
> + ret = PTR_ERR(cqctx->clock);
> + goto fail;
> + }
> +
> + return ret;
> +
> +fail:
> + cmdq_core_deinitialize(pdev);
> + return ret;
> +}
> +
> +static int cmdq_remove(struct platform_device *pdev)
> +{
> + cmdq_core_deinitialize(pdev);
> + return 0;
> +}
> +
> +static const struct of_device_id cmdq_of_ids[] = {
> + {.compatible = "mediatek,mt8173-gce",},
> + {}
> +};
> +
> +static struct platform_driver cmdq_drv = {
> + .probe = cmdq_probe,
> + .remove = cmdq_remove,
> + .driver = {
> + .name = CMDQ_DRIVER_DEVICE_NAME,
> + .owner = THIS_MODULE,
> + .of_match_table = cmdq_of_ids,
> + }
> +};
> +
> +builtin_platform_driver(cmdq_drv);
> diff --git a/include/soc/mediatek/cmdq.h b/include/soc/mediatek/cmdq.h
> new file mode 100644
> index 0000000..29931c9
> --- /dev/null
> +++ b/include/soc/mediatek/cmdq.h
> @@ -0,0 +1,211 @@
> +/*
> + * Copyright (c) 2015 MediaTek Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + */
> +
> +#ifndef __MTK_CMDQ_H__
> +#define __MTK_CMDQ_H__
> +
> +#include <linux/platform_device.h>
> +#include <linux/types.h>
> +
> +enum cmdq_eng {
> + CMDQ_ENG_DISP_AAL,
> + CMDQ_ENG_DISP_COLOR0,
> + CMDQ_ENG_DISP_COLOR1,
> + CMDQ_ENG_DISP_DPI0,
> + CMDQ_ENG_DISP_DSI0,
> + CMDQ_ENG_DISP_DSI1,
> + CMDQ_ENG_DISP_GAMMA,
> + CMDQ_ENG_DISP_OD,
> + CMDQ_ENG_DISP_OVL0,
> + CMDQ_ENG_DISP_OVL1,
> + CMDQ_ENG_DISP_PWM0,
> + CMDQ_ENG_DISP_PWM1,
> + CMDQ_ENG_DISP_RDMA0,
> + CMDQ_ENG_DISP_RDMA1,
> + CMDQ_ENG_DISP_RDMA2,
> + CMDQ_ENG_DISP_UFOE,
> + CMDQ_ENG_DISP_WDMA0,
> + CMDQ_ENG_DISP_WDMA1,
> + CMDQ_ENG_MAX,
> +};
> +
> +/* events for CMDQ and display */
> +enum cmdq_event {
> + /* Display start of frame(SOF) events */
> + CMDQ_EVENT_DISP_OVL0_SOF = 11,
> + CMDQ_EVENT_DISP_OVL1_SOF = 12,
> + CMDQ_EVENT_DISP_RDMA0_SOF = 13,
> + CMDQ_EVENT_DISP_RDMA1_SOF = 14,
> + CMDQ_EVENT_DISP_RDMA2_SOF = 15,
> + CMDQ_EVENT_DISP_WDMA0_SOF = 16,
> + CMDQ_EVENT_DISP_WDMA1_SOF = 17,
> + /* Display end of frame(EOF) events */
> + CMDQ_EVENT_DISP_OVL0_EOF = 39,
> + CMDQ_EVENT_DISP_OVL1_EOF = 40,
> + CMDQ_EVENT_DISP_RDMA0_EOF = 41,
> + CMDQ_EVENT_DISP_RDMA1_EOF = 42,
> + CMDQ_EVENT_DISP_RDMA2_EOF = 43,
> + CMDQ_EVENT_DISP_WDMA0_EOF = 44,
> + CMDQ_EVENT_DISP_WDMA1_EOF = 45,
> + /* Mutex end of frame(EOF) events */
> + CMDQ_EVENT_MUTEX0_STREAM_EOF = 53,
> + CMDQ_EVENT_MUTEX1_STREAM_EOF = 54,
> + CMDQ_EVENT_MUTEX2_STREAM_EOF = 55,
> + CMDQ_EVENT_MUTEX3_STREAM_EOF = 56,
> + CMDQ_EVENT_MUTEX4_STREAM_EOF = 57,
> + /* Display underrun events */
> + CMDQ_EVENT_DISP_RDMA0_UNDERRUN = 63,
> + CMDQ_EVENT_DISP_RDMA1_UNDERRUN = 64,
> + CMDQ_EVENT_DISP_RDMA2_UNDERRUN = 65,
> + /* Keep this at the end of HW events */
> + CMDQ_MAX_HW_EVENT_COUNT = 260,
> + /* This is max event and also can be used as mask. */
> + CMDQ_SYNC_TOKEN_MAX = 0x1ff,
> + /* Invalid event */
> + CMDQ_SYNC_TOKEN_INVALID = -1,
> +};
> +
> +/* called after isr done or task done */
> +typedef int (*cmdq_async_flush_cb)(void *data);
> +
> +struct cmdq_task;
> +struct cmdq;
> +
> +struct cmdq_rec {
> + struct cmdq *cqctx;
> + u64 engine_flag;
> + size_t block_size; /* command size */
> + void *buf_ptr;
> + size_t buf_size;
> + /* running task after flush */
> + struct cmdq_task *running_task_ptr;
> + bool finalized;
> +};
> +
> +/**
> + * cmdq_rec_create() - create command queue record handle
> + * @dev: device
> + * @engine_flag: command queue engine flag
> + * @handle_ptr: command queue record handle pointer to retrieve cmdq_rec
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_create(struct device *dev, u64 engine_flag,
> + struct cmdq_rec **handle_ptr);
> +
> +/**
> + * cmdq_rec_reset() - reset command queue record commands
> + * @handle: the command queue record handle
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_reset(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_write() - append write command to the command queue record
> + * @handle: the command queue record handle
> + * @value: the specified target register value
> + * @addr: the specified target register physical address
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr);
> +
> +/**
> + * cmdq_rec_write_mask() - append write command with mask to the command queue
> + * record
> + * @handle: the command queue record handle
> + * @value: the specified target register value
> + * @addr: the specified target register physical address
> + * @mask: the specified target register mask
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
> + u32 addr, u32 mask);
> +
> +/**
> + * cmdq_rec_wait() - append wait command to the command queue record
> + * @handle: the command queue record handle
> + * @event: the desired event type to "wait and CLEAR"
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event);
> +
> +/**
> + * cmdq_rec_clear_event() - append clear event command to the command queue
> + * record
> + * @handle: the command queue record handle
> + * @event: the desired event to be cleared
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event);
> +
> +/**
> + * cmdq_rec_flush() - trigger CMDQ to execute the recorded commands
> + * @handle: the command queue record handle
> + *
> + * Return: 0 for success; else the error code is returned
> + *
> + * Trigger CMDQ to execute the recorded commands. Note that this is a
> + * synchronous flush function. When the function returned, the recorded
> + * commands have been done.
> + */
> +int cmdq_rec_flush(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_flush_async() - trigger CMDQ to asynchronously execute the
> + * recorded commands
> + * @handle: the command queue record handle
> + *
> + * Return: 0 for successfully start execution; else the error code is returned
> + *
> + * Trigger CMDQ to asynchronously execute the recorded commands. Note that this
> + * is an ASYNC function. When the function returned, it may or may not be
> + * finished. There is no way to retrieve the result.
> + */
> +int cmdq_rec_flush_async(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_flush_async_callback() - trigger CMDQ to asynchronously execute
> + * the recorded commands and call back after
> + * ISR is finished and this flush is finished
> + * @handle: the command queue record handle
> + * @isr_cb: called by ISR in the end of CMDQ ISR
> + * @isr_data: this data will pass back to isr_cb
> + * @done_cb: called after flush is done
> + * @done_data: this data will pass back to done_cb
> + *
> + * Return: 0 for success; else the error code is returned
> + *
> + * Trigger CMDQ to asynchronously execute the recorded commands and call back
> + * after ISR is finished and this flush is finished. Note that this is an ASYNC
> + * function. When the function returned, it may or may not be finished. The ISR
> + * callback function is called in the end of ISR, and the done callback
> + * function is called after all commands are done.
> + */
> +int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
> + cmdq_async_flush_cb isr_cb,
> + void *isr_data,
> + cmdq_async_flush_cb done_cb,
> + void *done_data);
> +
> +/**
> + * cmdq_rec_destroy() - destroy command queue record handle
> + * @handle: the command queue record handle
> + */
> +void cmdq_rec_destroy(struct cmdq_rec *handle);
> +
> +#endif /* __MTK_CMDQ_H__ */
More information about the Linux-mediatek
mailing list