[PATCH 7/7] media: rkvdec: hevc: check the RPS scratch allocation

Daniel Golle daniel at makrotopia.org
Sat Nov 14 02:23:22 PST 2026


rkvdec_hevc_prepare_hw_st_rps() uses the array returned by kzalloc()
without checking it, so a failed allocation while a decode job is
prepared dereferences a NULL pointer. Return -ENOMEM through
rkvdec_hevc_assemble_hw_rps() and let both back ends refuse the job
after completing the request's controls.

Fixes: c9a59dc2acc7 ("media: rkvdec: Add HEVC support for the VDPU381 variant")
Signed-off-by: Daniel Golle <daniel at makrotopia.org>
---
 .../rockchip/rkvdec/rkvdec-hevc-common.c      | 26 ++++++++++++++-----
 .../rockchip/rkvdec/rkvdec-hevc-common.h      |  4 +--
 .../rockchip/rkvdec/rkvdec-vdpu381-hevc.c     |  7 ++++-
 .../rockchip/rkvdec/rkvdec-vdpu383-hevc.c     |  7 ++++-
 4 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
index e3449ab0d415b..80df05bacaeba 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
@@ -329,24 +329,27 @@ static void st_ref_pic_set_calculate(struct rkvdec_hevc_run *run, int idx,
 	st_rps->num_delta_pocs = st_rps->num_positive_pics + st_rps->num_negative_pics;
 }
 
-static void rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
-					  struct rkvdec_hevc_st_rps_cache *cache)
+static int rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+					 struct rkvdec_hevc_st_rps_cache *cache)
 {
 	u8 num_sets = run->sps->num_short_term_ref_pic_sets;
 	size_t size;
 	int idx;
 
 	if (!run->ext_sps_st_rps)
-		return;
+		return 0;
 
 	size = num_sets * sizeof(*cache->sets);
 	if (cache->num_sets == num_sets && !memcmp(cache->sets, run->ext_sps_st_rps, size))
-		return;
+		return 0;
 
 	struct calculated_rps_st_set *calculated_rps_st_sets =
 		kzalloc(sizeof(struct calculated_rps_st_set) *
 			run->sps->num_short_term_ref_pic_sets, GFP_KERNEL);
 
+	if (!calculated_rps_st_sets)
+		return -ENOMEM;
+
 	for (idx = 0; idx < run->sps->num_short_term_ref_pic_sets; idx++) {
 		const struct v4l2_ctrl_hevc_ext_sps_st_rps *rps_data = &run->ext_sps_st_rps[idx];
 
@@ -362,13 +365,22 @@ static void rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rk
 
 	memcpy(cache->sets, run->ext_sps_st_rps, size);
 	cache->num_sets = num_sets;
+
+	return 0;
 }
 
-void rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
-				 struct rkvdec_hevc_st_rps_cache *st_cache)
+int rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+				struct rkvdec_hevc_st_rps_cache *st_cache)
 {
-	rkvdec_hevc_prepare_hw_st_rps(run, rps, st_cache);
+	int ret;
+
+	ret = rkvdec_hevc_prepare_hw_st_rps(run, rps, st_cache);
+	if (ret)
+		return ret;
+
 	rkvdec_hevc_assemble_hw_lt_rps(run, rps);
+
+	return 0;
 }
 
 struct vb2_buffer *
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
index f6a3dec6ff4ad..32f192743c8c8 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
@@ -71,8 +71,8 @@ void compute_tiles_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size,
 void compute_tiles_non_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size,
 			       u16 width, u16 height, s32 pic_in_cts_width,
 			       s32 pic_in_cts_height, u16 *column_width, u16 *row_height);
-void rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
-				 struct rkvdec_hevc_st_rps_cache *st_cache);
+int rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+				struct rkvdec_hevc_st_rps_cache *st_cache);
 void rkvdec_hevc_assemble_hw_scaling_list(struct rkvdec_ctx *ctx,
 					  struct rkvdec_hevc_run *run,
 					  struct scaling_factor *scaling_factor,
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
index 8388421a68a48..2a239114759a2 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
@@ -593,6 +593,7 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
 	struct rkvdec_hevc_run run;
 	struct rkvdec_hevc_ctx *hevc_ctx = ctx->priv;
 	struct rkvdec_hevc_priv_tbl *tbl = hevc_ctx->priv_tbl.cpu;
+	int ret;
 
 	rkvdec_hevc_run_preamble(ctx, &run);
 
@@ -606,7 +607,11 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
 	rkvdec_hevc_assemble_hw_scaling_list(ctx, &run, &tbl->scaling_list,
 					     &hevc_ctx->scaling_matrix_cache);
 	assemble_hw_pps(ctx, &run);
-	rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+	ret = rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+	if (ret) {
+		rkvdec_run_postamble(ctx, &run.base);
+		return ret;
+	}
 
 	config_registers(ctx, &run);
 
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
index 47f236295f1e5..52554c75f47f4 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
@@ -542,6 +542,7 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
 	struct rkvdec_hevc_ctx *hevc_ctx = ctx->priv;
 	struct rkvdec_hevc_priv_tbl *tbl = hevc_ctx->priv_tbl.cpu;
 	u32 timeout_threshold;
+	int ret;
 
 	rkvdec_hevc_run_preamble(ctx, &run);
 
@@ -560,7 +561,11 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
 	rkvdec_hevc_assemble_hw_scaling_list(ctx, &run, &tbl->scaling_list,
 					     &hevc_ctx->scaling_matrix_cache);
 	assemble_hw_pps(ctx, &run);
-	rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+	ret = rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+	if (ret) {
+		rkvdec_run_postamble(ctx, &run.base);
+		return ret;
+	}
 
 	config_registers(ctx, &run);
 
-- 
2.56.0



More information about the Linux-rockchip mailing list