avfilter/vf_colordetect: add support for full range alpha offset

This allows us to add a threshold for alpha comparisons to the full range
path as well.

The AArch64 changes were LLM-assisted, since I'm not familiar with that
architecture. However, the logic checks out to me.

Also adjusts the checkasm test to make sure there are no rounding differences
between the C and assembly implementations. Tested correct across 100k seeds
on x86.

checkasm:
 - CPU: AMD Ryzen 9 9950X3D 16-Core Processor (00B40F40)
 - Timing source: x86 (rdtsc)
 - Bench duration: 100000 µs per function (432788090 cycles)
 - Random seed: 4076092999

Benchmark results: (old)
  name                              cycles (vs ref)
  detect_alpha_8_full_c:            4670.7
  detect_alpha_8_full_avx2:          856.6 ( 5.45x)
  detect_alpha_8_full_avx512icl:     667.5 ( 7.00x)
  detect_alpha_16_full_c:           2705.7
  detect_alpha_16_full_avx2:         693.1 ( 3.90x)
  detect_alpha_16_full_avx512icl:    546.4 ( 4.94x)

Benchmark results: (new)
  name                              cycles (vs ref)
  detect_alpha_8_full_c:            7988.5
  detect_alpha_8_full_avx2:         1078.0 ( 7.41x)
  detect_alpha_8_full_avx512icl:     876.7 ( 9.11x)
  detect_alpha_16_full_c:           3734.7
  detect_alpha_16_full_avx2:         814.6 ( 4.58x)
  detect_alpha_16_full_avx512icl:    613.8 ( 6.08x)

Sponsored-by: nxtedition AB
Signed-off-by: Niklas Haas <git@haasn.dev>

libavfilter/aarch64/vf_colordetect_neon.S

@@ -177,13 +177,17 @@ endfunc
  * x4: ptrdiff_t width,
  * x5: ptrdiff_t height,
  * w6: int alpha_max,
+ * w7: int mpeg_range (unused)
+ * [sp]: int offset
  */
 function ff_detect_alpha_full_neon, export=1
+        ldr             w13, [sp]
         ands            x7, x4, #15             // width % 16
         bic             x8, x4, #15             // width / 16 * 16
         movi            v0.16b, #0
         movi            v1.16b, #255
         dup             v2.16b, w6              // alpha_max
+        dup             v18.16b, w13            // offset
         sub             x1, x1, x8              // color_stride - aligned_width
         sub             x3, x3, x8              // alpha_stride - aligned_width
         b.eq            1f
@@ -197,6 +201,7 @@ function ff_detect_alpha_full_neon, export=1
         ldr             q5, [x0], #16
         ldr             q6, [x2], #16
         subs            x12, x12, #16
+        uqsub           v5.16b, v5.16b, v18.16b // color - offset
         cmhi            v7.16b, v5.16b, v6.16b
         cmeq            v16.16b, v6.16b, v2.16b
         orr             v0.16b, v0.16b, v7.16b
@@ -207,6 +212,7 @@ function ff_detect_alpha_full_neon, export=1
         // handle loop tail
         ldr             q5, [x0]
         ldr             q6, [x2]
+        uqsub           v5.16b, v5.16b, v18.16b // color - offset
         cmhi            v7.16b, v5.16b, v6.16b
         cmeq            v16.16b, v6.16b, v2.16b
         and             v7.16b, v7.16b, v3.16b
@@ -242,13 +248,17 @@ endfunc
  * x4: ptrdiff_t width,
  * x5: ptrdiff_t height,
  * w6: int alpha_max,
+ * w7: int mpeg_range (unused)
+ * [sp]: int offset
  */
 function ff_detect_alpha16_full_neon, export=1
+        ldr             w13, [sp]
         ands            x7, x4, #7              // width % 8
         bic             x8, x4, #7              // width / 8 * 8
         movi            v0.8h, #0
         movi            v1.16b, #255
         dup             v2.8h, w6               // alpha_max
+        dup             v18.8h, w13             // offset
         sub             x1, x1, x8, lsl #1      // color_stride - (aligned_width * 2)
         sub             x3, x3, x8, lsl #1      // alpha_stride - (aligned_width * 2)
         b.eq            1f
@@ -262,6 +272,7 @@ function ff_detect_alpha16_full_neon, export=1
         ldr             q5, [x0], #16
         ldr             q6, [x2], #16
         subs            x12, x12, #8
+        uqsub           v5.8h, v5.8h, v18.8h    // color - offset
         cmhi            v7.8h, v5.8h, v6.8h
         cmeq            v16.8h, v6.8h, v2.8h
         orr             v0.16b, v0.16b, v7.16b
@@ -272,6 +283,7 @@ function ff_detect_alpha16_full_neon, export=1
         // handle loop tail
         ldr             q5, [x0]
         ldr             q6, [x2]
+        uqsub           v5.8h, v5.8h, v18.8h    // color - offset
         cmhi            v7.8h, v5.8h, v6.8h
         cmeq            v16.8h, v6.8h, v2.8h
         and             v7.16b, v7.16b, v3.16b

libavfilter/vf_colordetect.c

@@ -170,10 +170,14 @@ static int detect_alpha(AVFilterContext *ctx, void *arg,
      *
      * We subtract an additional offset of (1 << (depth - 1)) to account for
      * rounding errors in the value of `x`.
+     *
+     * For full range input this degenerates to `x > a`, so the offset is 0.
      */
     const int alpha_max = (1 << s->depth) - 1;
     const int mpeg_range = s->mpeg_max - s->mpeg_min;
-    const int offset = alpha_max * s->mpeg_min + (1 << (s->depth - 1));
+    int offset = alpha_max * s->mpeg_min + (1 << (s->depth - 1));
+    if (ctx->inputs[0]->color_range == AVCOL_RANGE_JPEG)
+        offset = 0;
 
     int ret = 0;
     for (int i = 0; i < nb_planes; i++) {

libavfilter/vf_colordetectdsp.h

@@ -119,7 +119,7 @@ ff_detect_alpha_full_c(const uint8_t *color, ptrdiff_t color_stride,
     while (height--) {
         uint8_t straight = 0;
         for (int x = 0; x < width; x++) {
-            straight  |= color[x] > alpha[x];
+            straight |= color[x] > alpha[x] + offset;
             transparent |= alpha[x] != alpha_max;
         }
         if (straight)
@@ -163,7 +163,7 @@ ff_detect_alpha16_full_c(const uint8_t *color, ptrdiff_t color_stride,
         const uint16_t *alpha16 = (const uint16_t *) alpha;
         uint8_t straight = 0;
         for (int x = 0; x < width; x++) {
-            straight  |= color16[x] > alpha16[x];
+            straight |= color16[x] > alpha16[x] + offset;
             transparent |= alpha16[x] != alpha_max;
         }
         if (straight)

libavfilter/x86/vf_colordetect.asm

@@ -92,6 +92,7 @@ cglobal detect_alpha%1_%3, 1, 6, 7, color, ret, alpha, x, width, height
     vpbroadcast%2 m5, r8m ; offset
 %else
     vpbroadcast%1 m3, r6m ; alpha_max
+    vpbroadcast%1 m4, r8m ; offset
 %endif
     mova m6, m3
     xor retd, retd
@@ -102,6 +103,7 @@ cglobal detect_alpha%1_%3, 1, 6, 7, color, ret, alpha, x, width, height
         movu m1, [colorq + xq]
         movu m2, [alphaq + xq]
         pand m6, m2
+        psubus%1 m1, m4
         pmaxu%1 m1, m2
     %else
         pmovzx%1%2 m1, [colorq + xq]

tests/checkasm/vf_colordetect.c

@@ -75,9 +75,16 @@ static void check_alpha_detect(int depth, enum AVColorRange range)
     const int mpeg_max = 235 << (depth - 8);
     const int alpha_max = (1 << depth) - 1;
     const int mpeg_range = mpeg_max - mpeg_min;
-    const int offset = alpha_max * mpeg_min + (1 << (depth - 1));
     int res_ref, res_new;
 
+    int offset;
+    int threshold = checkasm_rand() % FFMIN(HEIGHT, mpeg_min);
+    if (range == AVCOL_RANGE_JPEG) {
+        offset = threshold;
+    } else {
+        offset = alpha_max * (mpeg_min + threshold) + (1 << (depth - 1));
+    }
+
     FFColorDetectDSPContext dsp = {0};
     ff_color_detect_dsp_init(&dsp, depth, range);
 
@@ -89,13 +96,19 @@ static void check_alpha_detect(int depth, enum AVColorRange range)
     memset(luma,  0x80, HEIGHT * STRIDE);
     memset(alpha, 0xF0, HEIGHT * STRIDE);
 
-    /* Try and force overflow */
+    /* Try and force overflow and edge cases */
     if (depth > 8 && range == AVCOL_RANGE_MPEG) {
-        ((uint16_t *) luma)[0] = 235 << (depth - 8);
-        ((uint16_t *) luma)[1] =  16 << (depth - 8);
+        for (int i = 0; i < threshold; i++) {
+            uint16_t *line = (uint16_t *) (luma + i * STRIDE);
+            line[0] = (235 << (depth - 8)) + i;
+            line[1] = ( 16 << (depth - 8)) - i;
+        }
     } else {
-        luma[0] = 235;
-        luma[1] = 16;
+        for (int i = 0; i < threshold; i++) {
+            uint8_t *line = luma + i * STRIDE;
+            line[0] = 235 + i;
+            line[1] = 16 - i;
+        }
     }
 
     /* Place an out-of-range value in a random position near the center */