diff --git a/libavcodec/vulkan/ffv1_common.glsl b/libavcodec/vulkan/ffv1_common.glsl index 22fef6c969..61e8446e9c 100644 --- a/libavcodec/vulkan/ffv1_common.glsl +++ b/libavcodec/vulkan/ffv1_common.glsl @@ -81,7 +81,11 @@ layout (push_constant, scalar) uniform pushConstants { bool remap_allowed; }; +#ifdef RC_SUBGROUP +#include "rangecoder_subgroup.glsl" +#else #include "rangecoder.glsl" +#endif #if !defined(RGB) #define TYPE int16_t @@ -202,13 +206,19 @@ const uint32_t log2_run[41] = { 24, }; +#ifdef DECODE +#define IMG_QUALI coherent readonly +VTYPE2 linecache; +#else +#define IMG_QUALI readonly shared VTYPE2 linecache; +#endif #ifdef RGB #define RGB_LBUF (rgb_linecache - 1) #define LADDR(p) (ivec2((p).x, ((p).y & RGB_LBUF))) -ivec4 get_top(readonly uimage2D pred, ivec2 sp, ivec2 off, +ivec4 get_top(IMG_QUALI uimage2D pred, ivec2 sp, ivec2 off, uint comp, int sw, bool extend_lookup) { ivec2 yoff_border1 = expectEXT(off.x == 0, false) ? off + ivec2(1, -1) : off; @@ -235,7 +245,7 @@ ivec4 get_top(readonly uimage2D pred, ivec2 sp, ivec2 off, #define LADDR(p) (p) -ivec4 get_top(readonly uimage2D pred, ivec2 sp, ivec2 off, +ivec4 get_top(IMG_QUALI uimage2D pred, ivec2 sp, ivec2 off, uint comp, int sw, bool extend_lookup) { ivec2 yoff_border1 = off.x == 0 ? ivec2(1, -1) : ivec2(0, 0); @@ -271,7 +281,7 @@ ivec3 get_pred_top_quant(ivec4 top, uint8_t quant_table_idx, bool extend_lookup) return ivec3(top[0], top[1], base); } -ivec3 get_pred_top(readonly uimage2D pred, ivec2 sp, ivec2 off, +ivec3 get_pred_top(IMG_QUALI uimage2D pred, ivec2 sp, ivec2 off, uint comp, int sw, uint8_t quant_table_idx, bool extend_lookup) { return get_pred_top_quant(get_top(pred, sp, off, comp, sw, extend_lookup), @@ -296,15 +306,20 @@ ivec2 get_pred_left(ivec3 top, uint8_t quant_table_idx, bool extend_lookup) return ivec2(base, predict(cur, top.xy)); } -ivec2 get_pred(readonly uimage2D pred, ivec2 sp, ivec2 off, +ivec2 get_pred(IMG_QUALI uimage2D pred, ivec2 sp, ivec2 off, uint comp, int sw, uint8_t quant_table_idx, bool extend_lookup) { return get_pred_left(get_pred_top(pred, sp, off, comp, sw, quant_table_idx, extend_lookup), quant_table_idx, extend_lookup); } -void linecache_load(readonly uimage2D src, ivec2 sp, int y, uint comp) +void linecache_load(IMG_QUALI uimage2D src, ivec2 sp, int y, uint comp) { +#ifdef DECODE + linecache = VTYPE2(TYPE(0), TYPE(0)); + if (y > 0) + linecache[1] = TYPE(imageLoad(src, sp + LADDR(ivec2(0, y - 1)))[comp]); +#else if (gl_LocalInvocationID.x == 0) linecache[0] = TYPE(0); @@ -319,6 +334,7 @@ void linecache_load(readonly uimage2D src, ivec2 sp, int y, uint comp) } barrier(); +#endif } void linecache_next(TYPE cur) diff --git a/libavcodec/vulkan/ffv1_dec.comp.glsl b/libavcodec/vulkan/ffv1_dec.comp.glsl index 270d513b48..e5f2d79f1d 100644 --- a/libavcodec/vulkan/ffv1_dec.comp.glsl +++ b/libavcodec/vulkan/ffv1_dec.comp.glsl @@ -22,9 +22,11 @@ #pragma shader_stage(compute) #extension GL_GOOGLE_include_directive : require -#extension GL_KHR_shader_subgroup_ballot : require #define DECODE +#ifndef GOLOMB +#define RC_SUBGROUP +#endif #include "common.glsl" #include "ffv1_common.glsl" @@ -34,7 +36,7 @@ layout (set = 1, binding = 1, scalar) readonly buffer slice_offsets_buf { layout (set = 1, binding = 2, scalar) writeonly buffer slice_status_buf { uint32_t slice_status[]; }; -layout (set = 1, binding = 4) uniform uimage2D dec[]; +layout (set = 1, binding = 4) coherent uniform uimage2D dec[]; #ifdef FLOAT layout(set = 1, binding = 6) readonly buffer fltmap_buf { @@ -50,59 +52,8 @@ layout (set = 1, binding = 3, scalar) buffer slice_state_buf { layout (constant_id = 20) const bool quant_ballot = false; -#define READ(idx) get_rac_state(idx) -shared int sym_e; -shared bool rc_dec[CONTEXT_SIZE]; -int get_isymbol(void) -{ - sym_e = 0; - rc_dec[0] = true; - if (READ(0)) - return 0; - - int e = 1; - for (; e < 11; e++) { - rc_dec[e] = true; - if (!READ(e)) - break; - } - - int a = 1; - sym_e = e + 10; - rc_dec[sym_e] = true; - - if (c_bits > 10 && e == 11) { - do { - rc_state[10] = zero_one_state[rc_state[10] + 256]; - e++; - } while (READ(10)); - - a = READ(31) ? 0x3 : 0x2; - for (e -= 2; e >= 11; e--) { - rc_state[31] = zero_one_state[rc_state[31] + - (rc_data[31] ? 256 : 0)]; - a <<= 1; - a |= int(READ(31)); - } - - rc_dec[31] = true; - } - - e += 20; - for (; e >= 22; e--) { - a <<= 1; - a |= int(READ(e)); - rc_dec[e] = true; - } - - return READ(sym_e) ? -a : a; -} - void decode_line_pcm(ivec2 sp, int w, int y, int p) { - if (gl_LocalInvocationID.x > 0) - return; - #ifndef RGB if (p > 0 && p < 3) { w = ceil_rshift(w, chroma_shift.x); @@ -116,7 +67,8 @@ void decode_line_pcm(ivec2 sp, int w, int y, int p) for (uint i = (rct_offset >> 1); i > 0; i >>= 1) v |= get_rac_equi() ? i : 0; - imageStore(dec[p], sp + LADDR(ivec2(x, y)), uvec4(v)); + if (gl_LocalInvocationID.x == 0) + imageStore(dec[p], sp + LADDR(ivec2(x, y)), uvec4(v)); } } @@ -136,6 +88,7 @@ void decode_line(ivec2 sp, int w, bool ext = extend_lookup[quant_table_idx]; ivec2 qthr = quant_ballot ? quant_thresh[quant_table_idx][gl_LocalInvocationID.x] : ivec2(0); ivec2 qso = quant_ballot ? quant_scale_off[quant_table_idx] : ivec2(0); + uint sbase = state_off + gl_LocalInvocationID.x; for (int x = 0; x < w; x++) { ivec2 pr; @@ -151,29 +104,26 @@ void decode_line(ivec2 sp, int w, pr = get_pred(dec[p], sp, ivec2(x, y), 0, w, quant_table_idx, ext); } - uint rc_off = state_off + CONTEXT_SIZE*abs(pr[0]) + gl_LocalInvocationID.x; + int sgn = pr[0] < 0 ? -1 : 1; + uint soff = sbase + CONTEXT_SIZE*abs(pr[0]); + uint st = slice_rc_state[soff]; - rc_dec[gl_LocalInvocationID.x] = false; - rc_state[gl_LocalInvocationID.x] = slice_rc_state[rc_off]; - barrier(); + uint used, used_bits; + int v = get_isymbol(st, pr[1], sgn, used, used_bits); + uint vz = zero_extend(v, bits); + rac_check_window(); - if (gl_LocalInvocationID.x == 0) { - int diff = get_isymbol(); - if (pr[0] < 0) - diff = -diff; + if (subgroupInverseBallot(uvec4(used, 0, 0, 0))) + slice_rc_state[soff] = + zero_one_state[st + (subgroupInverseBallot(uvec4(used_bits, 0, 0, 0)) ? 256 : 0)]; - uint v = zero_extend(pr[1] + diff, bits); - imageStore(dec[p], sp + LADDR(ivec2(x, y)), uvec4(v)); - linecache_next(TYPE(v)); - } - - /* Image write now visible to other invocs */ - barrier(); - if (rc_dec[gl_LocalInvocationID.x]) - slice_rc_state[rc_off] = - zero_one_state[rc_state[gl_LocalInvocationID.x] + - (rc_data[gl_LocalInvocationID.x] ? 256 : 0)]; + if (gl_LocalInvocationID.x == 0) + imageStore(dec[p], sp + LADDR(ivec2(x, y)), uvec4(vz)); + linecache_next(TYPE(vz)); } + + memoryBarrierImage(); + barrier(); } #else /* GOLOMB */ @@ -473,13 +423,18 @@ void main(void) { uint slice_idx = gl_WorkGroupID.y*gl_NumWorkGroups.x + gl_WorkGroupID.x; - if (gl_LocalInvocationID.x == 0) - rc = slice_ctx[slice_idx].c; - barrier(); +#ifdef GOLOMB + rc = slice_ctx[slice_idx].c; +#else + rac_init_dec(slice_ctx[slice_idx].c); +#endif decode_slice(slice_ctx[slice_idx], slice_idx); if (gl_LocalInvocationID.x == 0) { +#ifndef GOLOMB + rc.bs_off += rc_pos; +#endif uint overread = 0; if (rc.bs_off >= (rc.bs_end + MAX_OVERREAD)) overread = rc.bs_off - rc.bs_end; diff --git a/libavcodec/vulkan/rangecoder_subgroup.glsl b/libavcodec/vulkan/rangecoder_subgroup.glsl new file mode 100644 index 0000000000..81937357f8 --- /dev/null +++ b/libavcodec/vulkan/rangecoder_subgroup.glsl @@ -0,0 +1,208 @@ +/* + * FFv1 codec + * + * Copyright (c) 2026 Lynne + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef VULKAN_RANGECODER_SUBGROUP_H +#define VULKAN_RANGECODER_SUBGROUP_H + +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_ballot : require + +#define CONTEXT_SIZE 32 +#define MAX_OVERREAD 2 + +#ifdef DECODE +#define RC_BTYPE readonly buffer +#else +#define RC_BTYPE uniform +#endif + +layout (set = 0, binding = 0, scalar) RC_BTYPE rangecoder_buf { + uint8_t rangecoder_state[512]; +}; + +shared uint8_t zero_one_state[512]; + +struct RangeCoder { + uint bs_start; + uint bs_off; + uint bs_end; + uint low; + uint range; + uint16_t outstanding_count; + int16_t outstanding_byte; +}; + +RangeCoder rc; + +uint rac_range1(uint range, uint state24) +{ + uint hi, lo; + umulExtended(range, state24, hi, lo); + return hi; +} + +#ifdef DECODE +uint rc_win; +uint rc_dist; +uint rc_next; +uint rc_pos; + +void rac_load_window(void) +{ + rc.bs_off += rc_pos; + rc_pos &= ~31u; + uint o = rc.bs_off + gl_SubgroupInvocationID; + rc_win = uint(~(o < rc.bs_end ? u8buf(uint64_t(slice_data) + o).v : uint8_t(0))); +} + +void rac_init_dec(in RangeCoder c) +{ + for (uint i = gl_SubgroupInvocationID; i < 512; i += gl_SubgroupSize) + zero_one_state[i] = rangecoder_state[i]; + barrier(); + + rc = c; + rc_dist = rc.range - rc.low - 1; + rc_pos = 0; + rac_load_window(); + rc_next = subgroupBroadcast(rc_win, 0); +} + +void rac_check_window(void) +{ + if (rc_pos > 10) + rac_load_window(); +} + +void refill(void) +{ + rc.range <<= 8; + rc_dist = (rc_dist << 8) | rc_next; + rc_next = subgroupBroadcast(rc_win, ++rc_pos); +} + +void rac_renorm(void) +{ + if (expectEXT(rc.range < 0x100, false)) + refill(); +} + +bool get_rac_internal(uint range1) +{ + bool bit = rc_dist < range1; + uint ranged = rc.range - range1; + uint distd = rc_dist - range1; + rc.range = bit ? range1 : ranged; + rc_dist = bit ? rc_dist : distd; + return bit; +} + +bool get_rac(uint state24) +{ + bool bit = get_rac_internal(rac_range1(rc.range, state24)); + rac_renorm(); + return bit; +} + +bool get_rac_equi(void) +{ + rac_check_window(); + bool bit = get_rac_internal(rc.range >> 1); + rac_renorm(); + return bit; +} + +const int AVERROR_INVALIDDATA = -0x41444E49; + +int get_isymbol_esc(inout uint st, uint sx, int pred, int sgn, out uint read, out uint bits) +{ + bool esc = c_bits > 10; + int n = 11; + if (esc) { + uint s10 = subgroupBroadcast(st, 10); + bool one; + [[dont_unroll]] do { + s10 = rangecoder_state[s10 + 256]; + n++; + rac_check_window(); + one = get_rac(s10 << 24); + } while (one && n < 33); + st = gl_SubgroupInvocationID == 10 ? s10 : st; + + if (one) { + read = 0x7FFu; + bits = 0x7FEu; + return pred + sgn*AVERROR_INVALIDDATA; + } + } + + uint s31 = subgroupBroadcast(st, 31); + rac_check_window(); + bool b = get_rac(s31 << 24); + uint a = b ? 0x3 : 0x2; + for (n -= 2; n >= 11; n--) { + s31 = rangecoder_state[s31 + (b ? 256 : 0)]; + rac_check_window(); + b = get_rac(s31 << 24); + a = (a << 1) | uint(b); + } + st = gl_SubgroupInvocationID == 31 ? s31 : st; + + rac_check_window(); + [[unroll]] for (int k = 8; k >= 0; k--) + a = (a << 1) | uint(get_rac(subgroupBroadcast(sx, 22 + k))); + + bool neg = get_rac(subgroupBroadcast(sx, 21)); + int sa = int(a)*sgn; + read = 0xFFE007FFu; + bits = ((esc ? 0x1FFu : 0x3FFu) << 1) | ((a & 0x3FFu) << 22) | (uint(neg) << 21); + return neg ? pred - sa : pred + sa; +} + +int get_isymbol(inout uint st, int pred, int sgn, out uint read, out uint bits) +{ + uint st24 = st << 24; + if (get_rac(subgroupBroadcast(st24, 0))) { + read = 1u; + bits = 1u; + return pred; + } + + int e = 1; + while (e < 11 && get_rac(subgroupBroadcast(st24, e))) + e++; + if (e == 11) + return get_isymbol_esc(st, st24, pred, sgn, read, bits); + + uint a = 1u; + for (int k = e - 2; k >= 0; k--) + a = (a << 1) | uint(get_rac(subgroupBroadcast(st24, 22 + k))); + bool neg = get_rac(subgroupBroadcast(st24, 10 + e)); + + read = (2u << e) - 1u + (((1u << (e - 1)) - 1u) << 22) + (1u << (10 + e)); + bits = (a << 22) + (1u << e) - 2u - (1u << (21 + e)) + (neg ? 1u << (10 + e) : 0u); + int sa = int(a)*sgn; + return neg ? pred - sa : pred + sa; +} +#endif + +#endif /* VULKAN_RANGECODER_SUBGROUP_H */ diff --git a/libavcodec/vulkan_ffv1.c b/libavcodec/vulkan_ffv1.c index fd272e5293..6f9527f17f 100644 --- a/libavcodec/vulkan_ffv1.c +++ b/libavcodec/vulkan_ffv1.c @@ -687,13 +687,14 @@ static int init_decode_shader(FFV1Context *f, FFVulkanContext *s, AVHWFramesContext *dec_frames_ctx, AVHWFramesContext *out_frames_ctx, VkSpecializationInfo *sl, int ac, int rgb, - int bayer, int ballot) + int bayer) { int err; uint32_t wg_x = ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 1; ff_vk_shader_load(shd, VK_SHADER_STAGE_COMPUTE_BIT, sl, - (uint32_t []) { wg_x, 1, 1 }, ballot ? CONTEXT_SIZE : 0); + (uint32_t []) { wg_x, 1, 1 }, + ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 0); ff_vk_shader_add_push_const(shd, 0, sizeof(FFv1ShaderParams), VK_SHADER_STAGE_COMPUTE_BIT); @@ -856,8 +857,6 @@ static void vk_decode_ffv1_uninit(FFVulkanDecodeShared *ctx) static int vk_decode_ffv1_init(AVCodecContext *avctx) { int err; - int ballot = 0; - FFv1QuantBallot qb; FFV1Context *f = avctx->priv_data; FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data; FFVulkanDecodeShared *ctx = NULL; @@ -878,6 +877,18 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx) return err; ctx = dec->shared_ctx; + /* Range coded slices are decoded by one subgroup of CONTEXT_SIZE + * invocations, each holding one of the states of a context */ + if (f->ac != AC_GOLOMB_RICE && + !(ctx->s.subgroup_props.minSubgroupSize <= CONTEXT_SIZE && + ctx->s.subgroup_props.maxSubgroupSize >= CONTEXT_SIZE && + (ctx->s.subgroup_props.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT))) { + av_log(avctx, AV_LOG_ERROR, "Range coded FFv1 decoding needs subgroups " + "of %i invocations\n", CONTEXT_SIZE); + err = AVERROR(ENOTSUP); + goto fail; + } + fv = ctx->sd_ctx = av_mallocz(sizeof(*fv)); if (!fv) { err = AVERROR(ENOMEM); @@ -908,15 +919,11 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx) if (f->ec && !!(avctx->err_recognition & AV_EF_CRCCHECK)) SPEC_LIST_ADD(sl, 1, 32, 1); - /* The ballot quantizer holds one threshold per invocation, so the - * workgroup must be a single subgroup of CONTEXT_SIZE invocations */ - if (f->ac != AC_GOLOMB_RICE && - ctx->s.subgroup_props.minSubgroupSize <= CONTEXT_SIZE && - ctx->s.subgroup_props.maxSubgroupSize >= CONTEXT_SIZE && - (ctx->s.subgroup_props.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT)) - ballot = ff_ffv1_vk_quant_ballot(f, &qb); - if (ballot) - SPEC_LIST_ADD(sl, 20, 32, 1); + if (f->ac != AC_GOLOMB_RICE) { + FFv1QuantBallot qb; + if (ff_ffv1_vk_quant_ballot(f, &qb)) + SPEC_LIST_ADD(sl, 20, 32, 1); + } /* Setup shader */ RET(init_setup_shader(f, &ctx->s, &ctx->exec_pool, &fv->setup, sl)); @@ -926,7 +933,7 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx) /* Decode shaders */ RET(init_decode_shader(f, &ctx->s, &ctx->exec_pool, &fv->decode, - dctx, hwfc, sl, f->ac, is_rgb, f->bayer, ballot)); + dctx, hwfc, sl, f->ac, is_rgb, f->bayer)); /* Init static data */ RET(ff_ffv1_vk_init_consts(&ctx->s, &fv->consts_buf, f));