From 2b99c068db09fb149330421d88999486fce83ed8 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 7 Oct 2026 00:16:06 +0200 Subject: [PATCH] avfilter/yadif: propagate errorsl Not a security issue. Not tested on VideoToolbox. Unprocessed output frames Replicated through UnModified FFmpeg with CUDA and Vulkan call failures injected by gdb Found-by: Joshua Rogers Suggested-by: Joshua Rogers --- libavfilter/vf_bwdif.c | 4 +++- libavfilter/vf_bwdif_cuda.c | 14 +++++++++----- libavfilter/vf_bwdif_vulkan.c | 7 +++++-- libavfilter/vf_yadif.c | 4 +++- libavfilter/vf_yadif_cuda.c | 14 +++++++++----- libavfilter/vf_yadif_videotoolbox.m | 10 ++++++---- libavfilter/yadif.h | 2 +- libavfilter/yadif_common.c | 28 +++++++++++++++++----------- 8 files changed, 53 insertions(+), 30 deletions(-) diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c index 5fba0e7a14..f7f715f3cb 100644 --- a/libavfilter/vf_bwdif.c +++ b/libavfilter/vf_bwdif.c @@ -108,7 +108,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) return 0; } -static void filter(AVFilterContext *ctx, AVFrame *dstpic, +static int filter(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff) { BWDIFContext *bwdif = ctx->priv; @@ -135,6 +135,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic, if (yadif->current_field == YADIF_FIELD_END) { yadif->current_field = YADIF_FIELD_NORMAL; } + + return 0; } static const enum AVPixelFormat pix_fmts[] = { diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c index cc954ab1d3..9879d12ab4 100644 --- a/libavfilter/vf_bwdif_cuda.c +++ b/libavfilter/vf_bwdif_cuda.c @@ -53,7 +53,7 @@ typedef struct DeintCUDAContext { #define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x) -static CUresult call_kernel(AVFilterContext *ctx, CUfunction func, +static int call_kernel(AVFilterContext *ctx, CUfunction func, CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next, CUarray_format format, int channels, int src_width, // Width is pixels per channel @@ -122,7 +122,7 @@ exit: return ret; } -static void filter(AVFilterContext *ctx, AVFrame *dst, +static int filter(AVFilterContext *ctx, AVFrame *dst, int parity, int tff) { DeintCUDAContext *s = ctx->priv; @@ -133,7 +133,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx)); if (ret < 0) - return; + return ret; for (i = 0; i < y->csp->nb_components; i++) { CUfunction func; @@ -151,6 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, channels = comp->step / pixel_size; if (pixel_size > 2 || channels > 2) { av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); + ret = AVERROR(EINVAL); goto exit; } switch (pixel_size) { @@ -164,6 +165,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, break; default: av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); + ret = AVERROR(EINVAL); goto exit; } @@ -172,7 +174,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, av_log(ctx, AV_LOG_TRACE, "Deinterlacing plane %d: pixel_size: %d channels: %d\n", comp->plane, pixel_size, channels); - call_kernel(ctx, func, + ret = call_kernel(ctx, func, (CUdeviceptr)y->prev->data[i], (CUdeviceptr)y->cur->data[i], (CUdeviceptr)y->next->data[i], @@ -185,6 +187,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0), dst->linesize[i] / comp->step, parity, tff, clip_max); + if (ret < 0) + goto exit; } if (y->current_field == YADIF_FIELD_END) { @@ -193,7 +197,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, exit: CHECK_CU(cu->cuCtxPopCurrent(&dummy)); - return; + return ret; } static av_cold void deint_cuda_uninit(AVFilterContext *ctx) diff --git a/libavfilter/vf_bwdif_vulkan.c b/libavfilter/vf_bwdif_vulkan.c index 2c1e4908ef..5f96d2f7a3 100644 --- a/libavfilter/vf_bwdif_vulkan.c +++ b/libavfilter/vf_bwdif_vulkan.c @@ -102,7 +102,7 @@ fail: return err; } -static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst, +static int bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst, int parity, int tff) { BWDIFVulkanContext *s = ctx->priv; @@ -112,13 +112,16 @@ static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst, .tff = tff, .current_field = y->current_field, }; + int ret; - ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst, + ret = ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst, (AVFrame *[]){ y->prev, y->cur, y->next }, 3, VK_NULL_HANDLE, 1, ¶ms, sizeof(params)); if (y->current_field == YADIF_FIELD_END) y->current_field = YADIF_FIELD_NORMAL; + + return ret; } static void bwdif_vulkan_uninit(AVFilterContext *avctx) diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c index 41ba056ad4..6fe9a97a13 100644 --- a/libavfilter/vf_yadif.c +++ b/libavfilter/vf_yadif.c @@ -225,7 +225,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) return 0; } -static void filter(AVFilterContext *ctx, AVFrame *dstpic, +static int filter(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff) { YADIFContext *yadif = ctx->priv; @@ -249,6 +249,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic, ff_filter_execute(ctx, filter_slice, &td, NULL, FFMIN(h, ff_filter_get_nb_threads(ctx))); } + + return 0; } static const enum AVPixelFormat pix_fmts[] = { diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c index 50ac61ad8a..435657de5d 100644 --- a/libavfilter/vf_yadif_cuda.c +++ b/libavfilter/vf_yadif_cuda.c @@ -53,7 +53,7 @@ typedef struct DeintCUDAContext { #define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x) -static CUresult call_kernel(AVFilterContext *ctx, CUfunction func, +static int call_kernel(AVFilterContext *ctx, CUfunction func, CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next, CUarray_format format, int channels, int src_width, // Width is pixels per channel @@ -121,7 +121,7 @@ exit: return ret; } -static void filter(AVFilterContext *ctx, AVFrame *dst, +static int filter(AVFilterContext *ctx, AVFrame *dst, int parity, int tff) { DeintCUDAContext *s = ctx->priv; @@ -132,7 +132,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx)); if (ret < 0) - return; + return ret; for (i = 0; i < y->csp->nb_components; i++) { CUfunction func; @@ -150,6 +150,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, channels = comp->step / pixel_size; if (pixel_size > 2 || channels > 2) { av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); + ret = AVERROR(EINVAL); goto exit; } switch (pixel_size) { @@ -163,12 +164,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, break; default: av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); + ret = AVERROR(EINVAL); goto exit; } av_log(ctx, AV_LOG_TRACE, "Deinterlacing plane %d: pixel_size: %d channels: %d\n", comp->plane, pixel_size, channels); - call_kernel(ctx, func, + ret = call_kernel(ctx, func, (CUdeviceptr)y->prev->data[i], (CUdeviceptr)y->cur->data[i], (CUdeviceptr)y->next->data[i], @@ -181,11 +183,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0), dst->linesize[i] / comp->step, parity, tff); + if (ret < 0) + goto exit; } exit: CHECK_CU(cu->cuCtxPopCurrent(&dummy)); - return; + return ret; } static av_cold void deint_cuda_uninit(AVFilterContext *ctx) diff --git a/libavfilter/vf_yadif_videotoolbox.m b/libavfilter/vf_yadif_videotoolbox.m index 9f09947cc2..b1152987cd 100644 --- a/libavfilter/vf_yadif_videotoolbox.m +++ b/libavfilter/vf_yadif_videotoolbox.m @@ -100,7 +100,7 @@ static void call_kernel(AVFilterContext *ctx, ff_objc_release(&buffer); } -static void filter(AVFilterContext *ctx, AVFrame *dst, +static int filter(AVFilterContext *ctx, AVFrame *dst, int parity, int tff) API_AVAILABLE(macos(10.11), ios(8.0)) { YADIFVTContext *s = ctx->priv; @@ -123,7 +123,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, channels = comp->step / pixel_size; if (pixel_size > 2 || channels > 2) { av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); - return; + return AVERROR(EINVAL); } switch (pixel_size) { case 1: @@ -134,7 +134,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, break; default: av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name); - return; + return AVERROR(EINVAL); } av_log(ctx, AV_LOG_TRACE, "Deinterlacing plane %d: pixel_size: %d channels: %d\n", @@ -151,7 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, if (cur) CFRelease(cur); if (next) CFRelease(next); if (dest) CFRelease(dest); - return; + return AVERROR_EXTERNAL; } tex_prev = CVMetalTextureGetTexture(prev); @@ -173,6 +173,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst, if (y->current_field == YADIF_FIELD_END) { y->current_field = YADIF_FIELD_NORMAL; } + + return 0; } static av_cold void do_uninit(AVFilterContext *ctx) API_AVAILABLE(macos(10.11), ios(8.0)) diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h index 888ba12365..21c6df29c2 100644 --- a/libavfilter/yadif.h +++ b/libavfilter/yadif.h @@ -62,7 +62,7 @@ typedef struct YADIFContext { AVFrame *prev; AVFrame *out; - void (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff); + int (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff); /** * Required alignment for filter_line diff --git a/libavfilter/yadif_common.c b/libavfilter/yadif_common.c index a6cd981684..2b1c9bc1cd 100644 --- a/libavfilter/yadif_common.c +++ b/libavfilter/yadif_common.c @@ -49,7 +49,11 @@ static int return_frame(AVFilterContext *ctx, int is_second) yadif->current_field = YADIF_FIELD_END; } - yadif->filter(ctx, yadif->out, tff ^ !is_second, tff); + ret = yadif->filter(ctx, yadif->out, tff ^ !is_second, tff); + if (ret < 0) { + av_frame_free(&yadif->out); + return ret; + } if (is_second) { int64_t cur_pts = yadif->cur->pts; @@ -100,13 +104,19 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame) { AVFilterContext *ctx = link->dst; YADIFContext *yadif = ctx->priv; + int ret; av_assert0(frame); ff_ccfifo_extract(&yadif->cc_fifo, frame); - if (yadif->frame_pending) - return_frame(ctx, 1); + if (yadif->frame_pending) { + ret = return_frame(ctx, 1); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } + } if (yadif->prev) av_frame_free(&yadif->prev); @@ -177,10 +187,8 @@ int ff_yadif_request_frame(AVFilterLink *link) YADIFContext *yadif = ctx->priv; int ret; - if (yadif->frame_pending) { - return_frame(ctx, 1); - return 0; - } + if (yadif->frame_pending) + return return_frame(ctx, 1); if (yadif->eof) return AVERROR_EOF; @@ -196,13 +204,11 @@ int ff_yadif_request_frame(AVFilterLink *link) yadif->current_field = YADIF_FIELD_BACK_END; next->pts = yadif->next->pts * 2 - yadif->cur->pts; - ff_yadif_filter_frame(ctx->inputs[0], next); + ret = ff_yadif_filter_frame(ctx->inputs[0], next); yadif->eof = 1; - } else if (ret < 0) { - return ret; } - return 0; + return ret; } int ff_yadif_config_output_common(AVFilterLink *outlink)