avfilter/yadif: propagate errorsl

Not a security issue. Not tested on VideoToolbox.

Unprocessed output frames Replicated through UnModified FFmpeg with CUDA and Vulkan call failures injected by gdb
Found-by: Joshua Rogers <joshua.rogers@aisle.com>
Suggested-by: Joshua Rogers <joshua.rogers@aisle.com>
This commit is contained in:
Michael Niedermayer authored and michaelni committed 2026-10-14 17:59:13 +00:00
1 parent bd3d2316c0
commit 2b99c068db
8 files changed
+53 -30

No files matched your search

+3 -1
View File
@@ -108,7 +108,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
return 0;
}
static void filter(AVFilterContext *ctx, AVFrame *dstpic,
static int filter(AVFilterContext *ctx, AVFrame *dstpic,
int parity, int tff)
{
BWDIFContext *bwdif = ctx->priv;
@@ -135,6 +135,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic,
if (yadif->current_field == YADIF_FIELD_END) {
yadif->current_field = YADIF_FIELD_NORMAL;
}
return 0;
}
static const enum AVPixelFormat pix_fmts[] = {
+9 -5
View File
@@ -53,7 +53,7 @@ typedef struct DeintCUDAContext {
#define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x)
static CUresult call_kernel(AVFilterContext *ctx, CUfunction func,
static int call_kernel(AVFilterContext *ctx, CUfunction func,
CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next,
CUarray_format format, int channels,
int src_width, // Width is pixels per channel
@@ -122,7 +122,7 @@ exit:
return ret;
}
static void filter(AVFilterContext *ctx, AVFrame *dst,
static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
DeintCUDAContext *s = ctx->priv;
@@ -133,7 +133,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx));
if (ret < 0)
return;
return ret;
for (i = 0; i < y->csp->nb_components; i++) {
CUfunction func;
@@ -151,6 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
ret = AVERROR(EINVAL);
goto exit;
}
switch (pixel_size) {
@@ -164,6 +165,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
ret = AVERROR(EINVAL);
goto exit;
}
@@ -172,7 +174,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
comp->plane, pixel_size, channels);
call_kernel(ctx, func,
ret = call_kernel(ctx, func,
(CUdeviceptr)y->prev->data[i],
(CUdeviceptr)y->cur->data[i],
(CUdeviceptr)y->next->data[i],
@@ -185,6 +187,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0),
dst->linesize[i] / comp->step,
parity, tff, clip_max);
if (ret < 0)
goto exit;
}
if (y->current_field == YADIF_FIELD_END) {
@@ -193,7 +197,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
exit:
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
return;
return ret;
}
static av_cold void deint_cuda_uninit(AVFilterContext *ctx)
+5 -2
View File
@@ -102,7 +102,7 @@ fail:
return err;
}
static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
static int bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
BWDIFVulkanContext *s = ctx->priv;
@@ -112,13 +112,16 @@ static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
.tff = tff,
.current_field = y->current_field,
};
int ret;
ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst,
ret = ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst,
(AVFrame *[]){ y->prev, y->cur, y->next }, 3,
VK_NULL_HANDLE, 1, &params, sizeof(params));
if (y->current_field == YADIF_FIELD_END)
y->current_field = YADIF_FIELD_NORMAL;
return ret;
}
static void bwdif_vulkan_uninit(AVFilterContext *avctx)
+3 -1
View File
@@ -225,7 +225,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
return 0;
}
static void filter(AVFilterContext *ctx, AVFrame *dstpic,
static int filter(AVFilterContext *ctx, AVFrame *dstpic,
int parity, int tff)
{
YADIFContext *yadif = ctx->priv;
@@ -249,6 +249,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic,
ff_filter_execute(ctx, filter_slice, &td, NULL,
FFMIN(h, ff_filter_get_nb_threads(ctx)));
}
return 0;
}
static const enum AVPixelFormat pix_fmts[] = {
+9 -5
View File
@@ -53,7 +53,7 @@ typedef struct DeintCUDAContext {
#define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x)
static CUresult call_kernel(AVFilterContext *ctx, CUfunction func,
static int call_kernel(AVFilterContext *ctx, CUfunction func,
CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next,
CUarray_format format, int channels,
int src_width, // Width is pixels per channel
@@ -121,7 +121,7 @@ exit:
return ret;
}
static void filter(AVFilterContext *ctx, AVFrame *dst,
static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
DeintCUDAContext *s = ctx->priv;
@@ -132,7 +132,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx));
if (ret < 0)
return;
return ret;
for (i = 0; i < y->csp->nb_components; i++) {
CUfunction func;
@@ -150,6 +150,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
ret = AVERROR(EINVAL);
goto exit;
}
switch (pixel_size) {
@@ -163,12 +164,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
ret = AVERROR(EINVAL);
goto exit;
}
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
comp->plane, pixel_size, channels);
call_kernel(ctx, func,
ret = call_kernel(ctx, func,
(CUdeviceptr)y->prev->data[i],
(CUdeviceptr)y->cur->data[i],
(CUdeviceptr)y->next->data[i],
@@ -181,11 +183,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0),
dst->linesize[i] / comp->step,
parity, tff);
if (ret < 0)
goto exit;
}
exit:
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
return;
return ret;
}
static av_cold void deint_cuda_uninit(AVFilterContext *ctx)
+6 -4
View File
@@ -100,7 +100,7 @@ static void call_kernel(AVFilterContext *ctx,
ff_objc_release(&buffer);
}
static void filter(AVFilterContext *ctx, AVFrame *dst,
static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff) API_AVAILABLE(macos(10.11), ios(8.0))
{
YADIFVTContext *s = ctx->priv;
@@ -123,7 +123,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
return;
return AVERROR(EINVAL);
}
switch (pixel_size) {
case 1:
@@ -134,7 +134,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
return;
return AVERROR(EINVAL);
}
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
@@ -151,7 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
if (cur) CFRelease(cur);
if (next) CFRelease(next);
if (dest) CFRelease(dest);
return;
return AVERROR_EXTERNAL;
}
tex_prev = CVMetalTextureGetTexture(prev);
@@ -173,6 +173,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
if (y->current_field == YADIF_FIELD_END) {
y->current_field = YADIF_FIELD_NORMAL;
}
return 0;
}
static av_cold void do_uninit(AVFilterContext *ctx) API_AVAILABLE(macos(10.11), ios(8.0))
+1 -1
View File
@@ -62,7 +62,7 @@ typedef struct YADIFContext {
AVFrame *prev;
AVFrame *out;
void (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff);
int (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff);
/**
* Required alignment for filter_line
+17 -11
View File
@@ -49,7 +49,11 @@ static int return_frame(AVFilterContext *ctx, int is_second)
yadif->current_field = YADIF_FIELD_END;
}
yadif->filter(ctx, yadif->out, tff ^ !is_second, tff);
ret = yadif->filter(ctx, yadif->out, tff ^ !is_second, tff);
if (ret < 0) {
av_frame_free(&yadif->out);
return ret;
}
if (is_second) {
int64_t cur_pts = yadif->cur->pts;
@@ -100,13 +104,19 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame)
{
AVFilterContext *ctx = link->dst;
YADIFContext *yadif = ctx->priv;
int ret;
av_assert0(frame);
ff_ccfifo_extract(&yadif->cc_fifo, frame);
if (yadif->frame_pending)
return_frame(ctx, 1);
if (yadif->frame_pending) {
ret = return_frame(ctx, 1);
if (ret < 0) {
av_frame_free(&frame);
return ret;
}
}
if (yadif->prev)
av_frame_free(&yadif->prev);
@@ -177,10 +187,8 @@ int ff_yadif_request_frame(AVFilterLink *link)
YADIFContext *yadif = ctx->priv;
int ret;
if (yadif->frame_pending) {
return_frame(ctx, 1);
return 0;
}
if (yadif->frame_pending)
return return_frame(ctx, 1);
if (yadif->eof)
return AVERROR_EOF;
@@ -196,13 +204,11 @@ int ff_yadif_request_frame(AVFilterLink *link)
yadif->current_field = YADIF_FIELD_BACK_END;
next->pts = yadif->next->pts * 2 - yadif->cur->pts;
ff_yadif_filter_frame(ctx->inputs[0], next);
ret = ff_yadif_filter_frame(ctx->inputs[0], next);
yadif->eof = 1;
} else if (ret < 0) {
return ret;
}
return 0;
return ret;
}
int ff_yadif_config_output_common(AVFilterLink *outlink)