avfilter/vf_fieldmatch: use the chroma subsampling when marking combed chroma

Fixes: out of array write
Fixes: z4YbWg6eX0Hr

Found during triage/review of the security report 0fHkOzVCAhre

Out of array write Replicated through UnModified FFmpeg with ASAN

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer authored and michaelni committed 2026-10-24 20:19:14 +00:00
1 parent cf725b9352
commit 95d27d9f05
1 file changed
+42 -30
+42 -30
View File
@@ -34,6 +34,7 @@
#include "libavutil/avassert.h"
#include "libavutil/imgutils.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/timestamp.h"
@@ -195,6 +196,40 @@ static void fill_buf(uint8_t *data, int w, int h, int linesize, uint8_t v)
}
}
static av_always_inline void mark_combed_chroma(const FieldMatchContext *fm, const AVFrame *src,
int hsub, int vsub)
{
const uint8_t *cmkpU = fm->cmask_data[1];
const uint8_t *cmkpV = fm->cmask_data[2];
const int width = AV_CEIL_RSHIFT(src->width, hsub);
const int height = AV_CEIL_RSHIFT(src->height, vsub);
const ptrdiff_t cmk_linesize = fm->cmask_linesize[0];
const ptrdiff_t cmk_linesizeUV = fm->cmask_linesize[2];
for (int y = 1; y < height - 1; y++) {
uint8_t *cmkp_row = fm->cmask_data[0] + ((y << vsub) - (vsub ? y & 1 : 0)) * cmk_linesize;
cmkpV += cmk_linesizeUV;
cmkpU += cmk_linesizeUV;
for (int x = 1; x < width - 1; x++) {
#define HAS_FF_AROUND(p, lz) (p[(x)-1 - (lz)] == 0xff || p[(x) - (lz)] == 0xff || p[(x)+1 - (lz)] == 0xff || \
p[(x)-1 ] == 0xff || p[(x)+1 ] == 0xff || \
p[(x)-1 + (lz)] == 0xff || p[(x) + (lz)] == 0xff || p[(x)+1 + (lz)] == 0xff)
if ((cmkpV[x] == 0xff && HAS_FF_AROUND(cmkpV, cmk_linesizeUV)) ||
(cmkpU[x] == 0xff && HAS_FF_AROUND(cmkpU, cmk_linesizeUV))) {
uint8_t *cmkp = cmkp_row + (x << hsub);
for (int i = 0; i < (1 << vsub) + !!vsub; i++, cmkp += cmk_linesize) {
if (hsub == 2)
AV_WN32A(cmkp, 0xffffffff);
else if (hsub)
AV_WN16A(cmkp, 0xffff);
else
*cmkp = 0xff;
}
}
}
}
}
static int calc_combed_score(const FieldMatchContext *fm, const AVFrame *src)
{
int x, y, plane, max_v = 0;
@@ -271,36 +306,13 @@ static int calc_combed_score(const FieldMatchContext *fm, const AVFrame *src)
}
if (fm->chroma) {
uint8_t *cmkp = fm->cmask_data[0];
uint8_t *cmkpU = fm->cmask_data[1];
uint8_t *cmkpV = fm->cmask_data[2];
const int width = AV_CEIL_RSHIFT(src->width, fm->hsub[INPUT_MAIN]);
const int height = AV_CEIL_RSHIFT(src->height, fm->vsub[INPUT_MAIN]);
const int cmk_linesize = fm->cmask_linesize[0] << 1;
const int cmk_linesizeUV = fm->cmask_linesize[2];
uint8_t *cmkpp = cmkp - (cmk_linesize>>1);
uint8_t *cmkpn = cmkp + (cmk_linesize>>1);
uint8_t *cmkpnn = cmkp + cmk_linesize;
for (y = 1; y < height - 1; y++) {
cmkpp += cmk_linesize;
cmkp += cmk_linesize;
cmkpn += cmk_linesize;
cmkpnn += cmk_linesize;
cmkpV += cmk_linesizeUV;
cmkpU += cmk_linesizeUV;
for (x = 1; x < width - 1; x++) {
#define HAS_FF_AROUND(p, lz) (p[(x)-1 - (lz)] == 0xff || p[(x) - (lz)] == 0xff || p[(x)+1 - (lz)] == 0xff || \
p[(x)-1 ] == 0xff || p[(x)+1 ] == 0xff || \
p[(x)-1 + (lz)] == 0xff || p[(x) + (lz)] == 0xff || p[(x)+1 + (lz)] == 0xff)
if ((cmkpV[x] == 0xff && HAS_FF_AROUND(cmkpV, cmk_linesizeUV)) ||
(cmkpU[x] == 0xff && HAS_FF_AROUND(cmkpU, cmk_linesizeUV))) {
((uint16_t*)cmkp)[x] = 0xffff;
((uint16_t*)cmkpn)[x] = 0xffff;
if (y&1) ((uint16_t*)cmkpp)[x] = 0xffff;
else ((uint16_t*)cmkpnn)[x] = 0xffff;
}
}
}
const int hsub = fm->hsub[INPUT_MAIN];
const int vsub = fm->vsub[INPUT_MAIN];
if (hsub == 0 && vsub == 0) mark_combed_chroma(fm, src, 0, 0);
else if (hsub == 1 && vsub == 0) mark_combed_chroma(fm, src, 1, 0);
else if (hsub == 1 && vsub == 1) mark_combed_chroma(fm, src, 1, 1);
else if (hsub == 2 && vsub == 0) mark_combed_chroma(fm, src, 2, 0);
else mark_combed_chroma(fm, src, hsub, vsub);
}
{