mirrors/ffmpeg
0
0 Fork 0
mirror of https://git.ffmpeg.org/ffmpeg.git synced 2026-10-28 10:04:37 +08:00
Code Activity
Files
0ae15fffefd66d78828902176da25d19ae652a0e
ffmpeg/libavcodec/vulkan_decode.c
T
Lynne 0ae15fffef vulkan_decode: do not bind the current picture twice when its slot is active 
When decoding the second field of a pair, the picture is already bound as
an active reference at its DPB slot, since the first field made it one.
Appending it again as an inactive slot duplicates the picture resource
within the coding scope, which is not allowed
(VUID-vkCmdBeginVideoCodingKHR-pPictureResource-07238).
2026-09-20 14:45:50 +09:00

1448 lines
57 KiB
C
Raw Blame History

/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/refstruct.h"
#include "vulkan_video.h"
#include "vulkan_decode.h"
#include "config_components.h"
#include "libavutil/avassert.h"
#include "libavutil/mem.h"
#include "libavutil/vulkan_loader.h"
#include "h264dec.h"
#define DECODER_IS_SDR(codec_id) \
(((codec_id) == AV_CODEC_ID_FFV1) || \
((codec_id) == AV_CODEC_ID_DPX) || \
((codec_id) == AV_CODEC_ID_APV) || \
((codec_id) == AV_CODEC_ID_PRORES_RAW) || \
((codec_id) == AV_CODEC_ID_PRORES))
#if CONFIG_H264_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
#endif
#if CONFIG_HEVC_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_hevc_desc;
#endif
#if CONFIG_VP9_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_vp9_desc;
#endif
#if CONFIG_AV1_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_av1_desc;
#endif
#if CONFIG_FFV1_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_ffv1_desc;
#endif
#if CONFIG_PRORES_RAW_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_prores_raw_desc;
#endif
#if CONFIG_PRORES_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_prores_desc;
#endif
#if CONFIG_DPX_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_dpx_desc;
#endif
#if CONFIG_APV_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_apv_desc;
#endif
static const FFVulkanDecodeDescriptor *dec_descs[] = {
#if CONFIG_H264_VULKAN_HWACCEL
&ff_vk_dec_h264_desc,
#endif
#if CONFIG_HEVC_VULKAN_HWACCEL
&ff_vk_dec_hevc_desc,
#endif
#if CONFIG_VP9_VULKAN_HWACCEL
&ff_vk_dec_vp9_desc,
#endif
#if CONFIG_AV1_VULKAN_HWACCEL
&ff_vk_dec_av1_desc,
#endif
#if CONFIG_FFV1_VULKAN_HWACCEL
&ff_vk_dec_ffv1_desc,
#endif
#if CONFIG_PRORES_RAW_VULKAN_HWACCEL
&ff_vk_dec_prores_raw_desc,
#endif
#if CONFIG_PRORES_VULKAN_HWACCEL
&ff_vk_dec_prores_desc,
#endif
#if CONFIG_DPX_VULKAN_HWACCEL
&ff_vk_dec_dpx_desc,
#endif
#if CONFIG_APV_VULKAN_HWACCEL
&ff_vk_dec_apv_desc,
#endif
};
typedef struct FFVulkanDecodeProfileData {
VkVideoDecodeH264ProfileInfoKHR h264_profile;
VkVideoDecodeH265ProfileInfoKHR h265_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
VkVideoDecodeVP9ProfileInfoKHR vp9_profile;
#endif
VkVideoDecodeAV1ProfileInfoKHR av1_profile;
VkVideoDecodeUsageInfoKHR usage;
VkVideoProfileInfoKHR profile;
VkVideoProfileListInfoKHR profile_list;
} FFVulkanDecodeProfileData;
static const FFVulkanDecodeDescriptor *get_codecdesc(enum AVCodecID codec_id)
{
for (size_t i = 0; i < FF_ARRAY_ELEMS(dec_descs); i++)
if (dec_descs[i]->codec_id == codec_id)
return dec_descs[i];
av_assert1(!"no codec descriptor");
return NULL;
}
static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
{
const VkVideoProfileListInfoKHR *profile_list;
VkStructureType profile_struct_type =
codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
#if CONFIG_VP9_VULKAN_HWACCEL
codec_id == AV_CODEC_ID_VP9 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_PROFILE_INFO_KHR :
#endif
codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
VK_STRUCTURE_TYPE_MAX_ENUM;
if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
return NULL;
profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
if (!profile_list)
return NULL;
for (int i = 0; i < profile_list->profileCount; i++)
if (ff_vk_find_struct(profile_list->pProfiles[i].pNext, profile_struct_type))
return &profile_list->pProfiles[i];
return NULL;
}
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
{
FFVulkanDecodeContext *src_ctx = src->internal->hwaccel_priv_data;
FFVulkanDecodeContext *dst_ctx = dst->internal->hwaccel_priv_data;
av_refstruct_replace(&dst_ctx->shared_ctx, src_ctx->shared_ctx);
av_refstruct_replace(&dst_ctx->session_params, src_ctx->session_params);
dst_ctx->dedicated_dpb = src_ctx->dedicated_dpb;
dst_ctx->external_fg = src_ctx->external_fg;
return 0;
}
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
{
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
av_refstruct_unref(&dec->session_params);
return 0;
}
static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
{
FFVulkanDecodeShared *ctx = dec->shared_ctx;
FFVulkanFunctions *vk = &ctx->s.vkfn;
vkpic->dpb_frame = NULL;
vkpic->dpb_img = NULL;
vkpic->out_views = NULL;
vkpic->view.ref = VK_NULL_HANDLE;
vkpic->view.out = VK_NULL_HANDLE;
vkpic->invalidate_memory_ranges = vk->InvalidateMappedMemoryRanges;
}
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
FFVulkanDecodePicture *vkpic, int is_current,
int alloc_dpb)
{
int err;
FFVulkanDecodeShared *ctx = dec->shared_ctx;
vkpic->slices_size = 0;
/* If the decoder made a blank frame to make up for a missing ref, or the
* frame is the current frame so it's missing one, create a re-representation */
if (vkpic->view.ref)
return 0;
init_frame(dec, vkpic);
/* Refcounted, so executions keep it alive past the picture */
vkpic->out_views = ff_vk_imageviews_alloc(&ctx->s, 1);
if (!vkpic->out_views)
return AVERROR(ENOMEM);
if (ctx->common.layered_dpb && alloc_dpb) {
vkpic->view.ref = ctx->common.layered_view;
vkpic->view.aspect_ref = ctx->common.layered_aspect;
} else if (alloc_dpb) {
vkpic->dpb_img = av_refstruct_pool_get(ctx->common.dpb->img_pool);
if (!vkpic->dpb_img)
return AVERROR(ENOMEM);
/* The view is owned by the pool entry */
vkpic->view.ref = vkpic->dpb_img->view;
vkpic->view.aspect_ref = vkpic->dpb_img->aspect;
}
if (!alloc_dpb || is_current) {
AVHWFramesContext *frames = (AVHWFramesContext *)pic->hw_frames_ctx->data;
AVVulkanFramesContext *hwfc = frames->hwctx;
err = ff_vk_create_view(&ctx->s,
&vkpic->out_views->views[0], &vkpic->view.aspect,
((AVVkFrame *)pic->data[0])->img[0],
hwfc->format[0],
VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
(hwfc->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR),
0);
// the above fixes VUID-VkVideoBeginCodingInfoKHR-slotIndex-07245
if (err < 0)
return err;
vkpic->view.out = vkpic->out_views->views[0];
if (!alloc_dpb) {
vkpic->view.ref = vkpic->view.out;
vkpic->view.aspect_ref = vkpic->view.aspect;
}
}
return 0;
}
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
const uint8_t *data, size_t size, int add_startcode,
uint32_t *nb_slices, const uint32_t **offsets)
{
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
FFVulkanDecodeShared *ctx = dec->shared_ctx;
static const uint8_t startcode_prefix[3] = { 0x0, 0x0, 0x1 };
const size_t startcode_len = add_startcode ? sizeof(startcode_prefix) : 0;
const int nb = nb_slices ? *nb_slices : 0;
uint8_t *slices;
uint32_t *slice_off;
FFVkBuffer *vkbuf;
size_t new_size = vp->slices_size + startcode_len + size +
ctx->caps.minBitstreamBufferSizeAlignment;
new_size = FFALIGN(new_size, ctx->caps.minBitstreamBufferSizeAlignment);
if (offsets) {
slice_off = av_fast_realloc(dec->slice_off, &dec->slice_off_max,
(nb + 1)*sizeof(*slice_off));
if (!slice_off)
return AVERROR(ENOMEM);
*offsets = dec->slice_off = slice_off;
slice_off[nb] = vp->slices_size;
}
vkbuf = vp->slices_buf;
if (!vkbuf || vkbuf->size < new_size) {
int err;
FFVkBuffer *new_buf;
/* No point in requesting anything smaller. */
size_t buf_size = FFMAX(new_size, 1024*1024);
/* Align buffer to nearest power of two. Makes fragmentation management
* easier, and gives us ample headroom. */
buf_size = 2 << av_log2(buf_size);
/* When the frames context uses DRM modifier tiling,
* hwctx->create_pnext contains VkImageDrmFormatModifierListCreateInfoEXT,
* which per spec does not extend VkBufferCreateInfo, so we need to find
* the VkVideoProfileListInfoKHR structure within it. */
void *buf_pnext = ctx->s.hwfc->create_pnext;
if (ctx->s.hwfc->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
buf_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
VkBufferUsageFlags buf_usage = DECODER_IS_SDR(avctx->codec_id) ?
(VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR;
err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_buf,
buf_usage, buf_pnext, buf_size,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
if (err < 0)
err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_buf,
buf_usage, buf_pnext, buf_size,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
if (err < 0)
return err;
/* Copy data from the old buffer */
if (vkbuf) {
memcpy(new_buf->mapped_mem, vkbuf->mapped_mem, vp->slices_size);
av_refstruct_unref(&vp->slices_buf);
}
vp->slices_buf = new_buf;
vkbuf = new_buf;
}
slices = vkbuf->mapped_mem;
/* Startcode */
memcpy(slices + vp->slices_size, startcode_prefix, startcode_len);
/* Slice data */
memcpy(slices + vp->slices_size + startcode_len, data, size);
if (nb_slices)
*nb_slices = nb + 1;
vp->slices_size += startcode_len + size;
return 0;
}
static int create_empty_session_parameters(AVCodecContext *avctx,
FFVulkanDecodeShared *ctx,
VkVideoSessionParametersKHR *empty_session_params)
{
if (avctx->codec_id == AV_CODEC_ID_VP9)
return 0;
VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
};
VkVideoDecodeH265SessionParametersCreateInfoKHR h265_params = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_CREATE_INFO_KHR,
};
StdVideoAV1SequenceHeader av1_empty_seq = { 0 };
VkVideoDecodeAV1SessionParametersCreateInfoKHR av1_params = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_KHR,
.pStdSequenceHeader = &av1_empty_seq,
};
VkVideoSessionParametersCreateInfoKHR session_params_create = {
.sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
.pNext = avctx->codec_id == AV_CODEC_ID_H264 ? (void *)&h264_params :
avctx->codec_id == AV_CODEC_ID_HEVC ? (void *)&h265_params :
avctx->codec_id == AV_CODEC_ID_AV1 ? (void *)&av1_params :
NULL,
.videoSession = ctx->common.session,
};
VkResult ret;
FFVulkanContext *s = &ctx->s;
FFVulkanFunctions *vk = &s->vkfn;
ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
s->hwctx->alloc, empty_session_params);
if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
ff_vk_ret2str(ret));
return AVERROR_EXTERNAL;
}
return 0;
}
static int decode_reset(AVCodecContext *avctx, FFVulkanDecodeShared *ctx)
{
int err;
FFVulkanContext *s = &ctx->s;
FFVulkanFunctions *vk = &s->vkfn;
VkVideoBeginCodingInfoKHR decode_start = {
.sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
.videoSession = ctx->common.session,
};
if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)) {
err = create_empty_session_parameters(avctx, ctx,
&decode_start.videoSessionParameters);
if (err < 0)
return err;
}
VkVideoCodingControlInfoKHR decode_ctrl = {
.sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
.flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR,
};
VkVideoEndCodingInfoKHR decode_end = {
.sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
};
FFVkExecContext *exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
err = ff_vk_exec_start(&ctx->s, exec);
if (err < 0)
return err;
vk->CmdBeginVideoCodingKHR(exec->buf, &decode_start);
vk->CmdControlVideoCodingKHR(exec->buf, &decode_ctrl);
vk->CmdEndVideoCodingKHR(exec->buf, &decode_end);
err = ff_vk_exec_submit(&ctx->s, exec);
if (decode_start.videoSessionParameters) {
/* Wait to complete to delete the temporary session parameters */
if (err >= 0)
ff_vk_exec_wait(&ctx->s, exec);
vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
decode_start.videoSessionParameters,
s->hwctx->alloc);
}
if (err < 0)
av_log(avctx, AV_LOG_ERROR, "Unable to reset decoder: %s",
ff_vk_ret2str(err));
return err;
}
int ff_vk_decode_frame(AVCodecContext *avctx,
AVFrame *pic, FFVulkanDecodePicture *vp,
AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
{
int err;
VkResult ret;
VkCommandBuffer cmd_buf;
FFVkBuffer *sd_buf;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
FFVulkanDecodeShared *ctx = dec->shared_ctx;
FFVulkanFunctions *vk = &ctx->s.vkfn;
/* Output */
AVVkFrame *vkf = (AVVkFrame *)pic->buf[0]->data;
/* Quirks */
const int layered_dpb = ctx->common.layered_dpb;
VkVideoBeginCodingInfoKHR decode_start = {
.sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
.videoSession = ctx->common.session,
.videoSessionParameters = dec->session_params ?
*dec->session_params : VK_NULL_HANDLE,
.referenceSlotCount = vp->decode_info.referenceSlotCount,
.pReferenceSlots = vp->decode_info.pReferenceSlots,
};
VkVideoEndCodingInfoKHR decode_end = {
.sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
};
VkImageMemoryBarrier2 img_bar[38];
int nb_img_bar = 0;
size_t data_size = FFALIGN(vp->slices_size,
ctx->caps.minBitstreamBufferSizeAlignment);
FFVkExecContext *exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
/* The current picture's resource has to be bound as an inactive
* reference, unless its slot is already bound as an active one, as when
* decoding the second field of a pair. */
int cur_bound = 0;
for (int i = 0; i < decode_start.referenceSlotCount; i++)
cur_bound |= decode_start.pReferenceSlots[i].slotIndex == vp->ref_slot.slotIndex;
if (!cur_bound) {
VkVideoReferenceSlotInfoKHR *cur_vk_ref;
cur_vk_ref = (void *)&decode_start.pReferenceSlots[decode_start.referenceSlotCount];
cur_vk_ref[0] = vp->ref_slot;
cur_vk_ref[0].slotIndex = -1;
decode_start.referenceSlotCount++;
}
sd_buf = vp->slices_buf;
/* Flush if needed */
if (!(sd_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
VkMappedMemoryRange flush_buf = {
.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
.memory = sd_buf->mem,
.offset = 0,
.size = FFALIGN(vp->slices_size,
ctx->s.props.properties.limits.nonCoherentAtomSize),
};
ret = vk->FlushMappedMemoryRanges(ctx->s.hwctx->act_dev, 1, &flush_buf);
if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Failed to flush memory: %s\n",
ff_vk_ret2str(ret));
return AVERROR_EXTERNAL;
}
}
vp->decode_info.srcBuffer = sd_buf->buf;
vp->decode_info.srcBufferOffset = 0;
vp->decode_info.srcBufferRange = data_size;
/* Start command buffer recording */
err = ff_vk_exec_start(&ctx->s, exec);
if (err < 0)
return err;
cmd_buf = exec->buf;
/* Slices; owned by the execution from now on */
ff_vk_exec_move_dep_refstruct(&ctx->s, exec, &vp->slices_buf);
/* Parameters; unset when inline session parameters are used */
if (dec->session_params)
ff_vk_exec_add_dep_refstruct(&ctx->s, exec, dec->session_params);
err = ff_vk_exec_add_dep_frame(&ctx->s, exec, pic,
VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
if (err < 0) {
ff_vk_exec_discard(&ctx->s, exec);
return err;
}
/* The output view is kept alive by the execution context; freeing the
* picture then needs no host wait. */
ff_vk_exec_add_dep_refstruct(&ctx->s, exec, vp->out_views);
/* Output image - change layout, as it comes from a pool */
img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
.pNext = NULL,
.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.srcAccessMask = VK_ACCESS_2_NONE,
.dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
.oldLayout = vkf->layout[0],
.newLayout = (layered_dpb || vp->dpb_img) ?
VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR :
VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */
.srcQueueFamilyIndex = vkf->queue_family[0],
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = vkf->img[0],
.subresourceRange = (VkImageSubresourceRange) {
.aspectMask = vp->view.aspect,
.layerCount = 1,
.levelCount = 1,
},
};
ff_vk_exec_update_frame(&ctx->s, exec, pic,
&img_bar[nb_img_bar], &nb_img_bar);
/* Current picture's DISTINCT-mode DPB image: transition on first use */
if (vp->dpb_img) {
FFVkVideoDPBImage *di = vp->dpb_img;
if (di->layout != VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR) {
img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_NONE,
.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.srcAccessMask = VK_ACCESS_2_NONE,
.dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
.oldLayout = di->layout,
.newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = di->img,
.subresourceRange = (VkImageSubresourceRange) {
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.layerCount = 1,
.levelCount = 1,
},
};
nb_img_bar++;
di->layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
}
}
/* COINCIDE-mode references only: DISTINCT-mode references are covered
* by the memory barrier, with pool-owned views */
if (!dec->dedicated_dpb) {
for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
AVFrame *ref_frame = rpic[i];
FFVulkanDecodePicture *rvp = rvkp[i];
err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref_frame,
VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
if (err < 0) {
ff_vk_exec_discard(&ctx->s, exec);
return err;
}
/* The reference's image views are kept alive by the execution,
* so freeing the picture needs no host wait. */
if (err == 0 && rvp->out_views)
ff_vk_exec_add_dep_refstruct(&ctx->s, exec, rvp->out_views);
{
AVVkFrame *rvkf = (AVVkFrame *)ref_frame->data[0];
img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
.pNext = NULL,
.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.srcAccessMask = VK_ACCESS_2_NONE,
.dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
.oldLayout = rvkf->layout[0],
.newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
.srcQueueFamilyIndex = rvkf->queue_family[0],
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = rvkf->img[0],
.subresourceRange = (VkImageSubresourceRange) {
.aspectMask = rvp->view.aspect_ref,
.layerCount = 1,
.levelCount = 1,
},
};
ff_vk_exec_update_frame(&ctx->s, exec, ref_frame,
&img_bar[nb_img_bar], &nb_img_bar);
}
}
}
/* Internal DPB images are queue-exclusive: one decode-stage memory
* barrier per submission orders all prior work, references included, so
* they need no semaphores or dependencies, and image barriers only for
* initial layout transitions. */
VkMemoryBarrier2 mem_bar = {
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.srcAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
};
/* Change image layouts, and synchronize the internal DPB if in use */
vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
.pMemoryBarriers = &mem_bar,
.memoryBarrierCount = ctx->common.dpb ? 1 : 0,
.pImageMemoryBarriers = img_bar,
.imageMemoryBarrierCount = nb_img_bar,
});
/* Start, use parameters, decode and end decoding */
vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
vk->CmdDecodeVideoKHR(cmd_buf, &vp->decode_info);
vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
/* End recording and submit for execution */
return ff_vk_exec_submit(&ctx->s, exec);
}
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
{
/* Free slices data */
av_refstruct_unref(&vp->slices_buf);
/* The views are kept alive by every execution referencing them, so no
* host wait is needed. */
av_refstruct_unref(&vp->out_views);
av_frame_free(&vp->dpb_frame);
/* No wait: pool reuse is ordered by the DPB barrier */
av_refstruct_unref(&vp->dpb_img);
}
static void free_common(AVRefStructOpaque unused, void *obj)
{
FFVulkanDecodeShared *ctx = obj;
FFVulkanContext *s = &ctx->s;
/* Wait on and free execution pool */
ff_vk_exec_pool_free(&ctx->s, &ctx->exec_pool);
av_refstruct_pool_uninit(&ctx->buf_pool);
ff_vk_video_common_uninit(s, &ctx->common);
if (ctx->sd_ctx_free)
ctx->sd_ctx_free(ctx);
ff_vk_uninit(s);
}
static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
{
int err;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
AVVulkanDeviceContext *hwctx = device->hwctx;
FFVulkanDecodeShared *ctx;
if (dec->shared_ctx)
return 0;
dec->shared_ctx = av_refstruct_alloc_ext(sizeof(*ctx), 0, NULL,
free_common);
if (!dec->shared_ctx)
return AVERROR(ENOMEM);
ctx = dec->shared_ctx;
ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
hwctx->nb_enabled_dev_extensions);
if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
av_refstruct_unref(&dec->shared_ctx);
return AVERROR(ENOSYS);
}
}
err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
if (err < 0) {
av_refstruct_unref(&dec->shared_ctx);
return err;
}
return 0;
}
static VkResult vulkan_setup_profile(AVCodecContext *avctx,
FFVulkanDecodeProfileData *prof,
AVVulkanDeviceContext *hwctx,
FFVulkanFunctions *vk,
const FFVulkanDecodeDescriptor *vk_desc,
VkVideoDecodeH264CapabilitiesKHR *h264_caps,
VkVideoDecodeH265CapabilitiesKHR *h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
VkVideoDecodeVP9CapabilitiesKHR *vp9_caps,
#endif
VkVideoDecodeAV1CapabilitiesKHR *av1_caps,
VkVideoCapabilitiesKHR *caps,
VkVideoDecodeCapabilitiesKHR *dec_caps,
int cur_profile)
{
VkVideoDecodeUsageInfoKHR *usage = &prof->usage;
VkVideoProfileInfoKHR *profile = &prof->profile;
VkVideoProfileListInfoKHR *profile_list = &prof->profile_list;
VkVideoDecodeH264ProfileInfoKHR *h264_profile = &prof->h264_profile;
VkVideoDecodeH265ProfileInfoKHR *h265_profile = &prof->h265_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
VkVideoDecodeVP9ProfileInfoKHR *vp9_profile = &prof->vp9_profile;
#endif
VkVideoDecodeAV1ProfileInfoKHR *av1_profile = &prof->av1_profile;
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
if (!desc)
return AVERROR(EINVAL);
if (avctx->codec_id == AV_CODEC_ID_H264) {
dec_caps->pNext = h264_caps;
usage->pNext = h264_profile;
h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
/* Vulkan transmits all the constrant_set flags, rather than wanting them
* merged in the profile IDC */
h264_profile->stdProfileIdc = cur_profile & ~(AV_PROFILE_H264_CONSTRAINED |
AV_PROFILE_H264_INTRA);
h264_profile->pictureLayout =
VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR;
const H264Context *h = avctx->priv_data;
if (h && h->ps.sps && !h->ps.sps->frame_mbs_only_flag)
h264_profile->pictureLayout =
VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
} else if (avctx->codec_id == AV_CODEC_ID_H265) {
dec_caps->pNext = h265_caps;
usage->pNext = h265_profile;
h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
h265_profile->stdProfileIdc = cur_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
} else if (avctx->codec_id == AV_CODEC_ID_VP9) {
dec_caps->pNext = vp9_caps;
usage->pNext = vp9_profile;
vp9_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_PROFILE_INFO_KHR;
vp9_profile->stdProfile = cur_profile;
#endif
} else if (avctx->codec_id == AV_CODEC_ID_AV1) {
dec_caps->pNext = av1_caps;
usage->pNext = av1_profile;
av1_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR;
av1_profile->stdProfile = cur_profile;
av1_profile->filmGrainSupport = !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN);
}
usage->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
profile->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
profile->pNext = usage;
profile->videoCodecOperation = vk_desc->decode_op;
profile->chromaSubsampling = ff_vk_subsampling_from_av_desc(desc);
profile->lumaBitDepth = ff_vk_depth_from_av_depth(desc->comp[0].depth);
profile->chromaBitDepth = profile->lumaBitDepth;
/* A pixel format without a video profile representation is not a valid
* profile to query the capabilities for */
if (profile->chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_INVALID_KHR ||
profile->lumaBitDepth == VK_VIDEO_COMPONENT_BIT_DEPTH_INVALID_KHR)
return VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR;
profile_list->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
profile_list->profileCount = 1;
profile_list->pProfiles = profile;
/* Get the capabilities of the decoder for the given profile */
caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
caps->pNext = dec_caps;
dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
/* dec_caps->pNext already filled in */
return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
caps);
}
static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref,
enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
FFVulkanDecodeProfileData *prof,
int *dpb_dedicate)
{
VkResult ret;
int max_level, base_profile, cur_profile;
const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
AVVulkanDeviceContext *hwctx = device->hwctx;
enum AVPixelFormat source_format;
enum AVPixelFormat best_format;
VkFormat best_vkfmt;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
FFVulkanDecodeShared *ctx = dec->shared_ctx;
FFVulkanFunctions *vk = &ctx->s.vkfn;
VkVideoCapabilitiesKHR *caps = &ctx->caps;
VkVideoDecodeCapabilitiesKHR *dec_caps = &ctx->dec_caps;
VkVideoDecodeH264CapabilitiesKHR h264_caps = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
};
VkVideoDecodeH265CapabilitiesKHR h265_caps = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
};
#if CONFIG_VP9_VULKAN_HWACCEL
VkVideoDecodeVP9CapabilitiesKHR vp9_caps = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_CAPABILITIES_KHR,
};
#endif
VkVideoDecodeAV1CapabilitiesKHR av1_caps = {
.sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_KHR,
};
VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
.pNext = &prof->profile_list,
};
VkVideoFormatPropertiesKHR *ret_info;
uint32_t nb_out_fmts = 0;
if (!(vk_desc->decode_extension & ctx->s.extensions)) {
av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
avcodec_get_name(avctx->codec_id));
return AVERROR(ENOSYS);
}
cur_profile = avctx->profile;
base_profile = avctx->codec_id == AV_CODEC_ID_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE :
avctx->codec_id == AV_CODEC_ID_H265 ? AV_PROFILE_HEVC_MAIN :
#if CONFIG_VP9_VULKAN_HWACCEL
avctx->codec_id == AV_CODEC_ID_VP9 ? STD_VIDEO_VP9_PROFILE_0 :
#endif
avctx->codec_id == AV_CODEC_ID_AV1 ? STD_VIDEO_AV1_PROFILE_MAIN :
0;
ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
&h264_caps,
&h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
&vp9_caps,
#endif
&av1_caps,
caps,
dec_caps,
cur_profile);
if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
avctx->flags & AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH &&
avctx->profile != base_profile) {
av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
"again with profile %s\n",
avcodec_get_name(avctx->codec_id),
avcodec_profile_name(avctx->codec_id, cur_profile),
avcodec_profile_name(avctx->codec_id, base_profile));
cur_profile = base_profile;
ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
&h264_caps,
&h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
&vp9_caps,
#endif
&av1_caps,
caps,
dec_caps,
cur_profile);
}
if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR ||
ret == VK_ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR) {
av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
"%s profile \"%s\" not supported!\n",
avcodec_get_name(avctx->codec_id),
avcodec_profile_name(avctx->codec_id, cur_profile));
return AVERROR(EINVAL);
} else if (ret == VK_ERROR_VIDEO_PICTURE_LAYOUT_NOT_SUPPORTED_KHR) {
if (avctx->codec_id == AV_CODEC_ID_H264)
av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
"pictureLayout %#x not supported!\n",
(unsigned)prof->h264_profile.pictureLayout);
else
av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
"pictureLayout not supported!\n");
return AVERROR(EINVAL);
} else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
"format (%s) not supported!\n",
av_get_pix_fmt_name(avctx->sw_pix_fmt));
return AVERROR(EINVAL);
} else if (ret == VK_ERROR_FEATURE_NOT_PRESENT ||
ret == VK_ERROR_FORMAT_NOT_SUPPORTED) {
return AVERROR(EINVAL);
} else if (ret != VK_SUCCESS) {
return AVERROR_EXTERNAL;
}
max_level = avctx->codec_id == AV_CODEC_ID_H264 ? ff_vk_h264_level_to_av(h264_caps.maxLevelIdc) :
avctx->codec_id == AV_CODEC_ID_H265 ? ff_vk_h265_level_to_av(h265_caps.maxLevelIdc) :
#if CONFIG_VP9_VULKAN_HWACCEL
avctx->codec_id == AV_CODEC_ID_VP9 ? vp9_caps.maxLevel :
#endif
avctx->codec_id == AV_CODEC_ID_AV1 ? av1_caps.maxLevel :
0;
av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
avcodec_get_name(avctx->codec_id),
avcodec_profile_name(avctx->codec_id, cur_profile));
av_log(avctx, AV_LOG_VERBOSE, " Maximum level: %i (stream %i)\n",
max_level, avctx->level);
av_log(avctx, AV_LOG_VERBOSE, " Width: from %i to %i\n",
caps->minCodedExtent.width, caps->maxCodedExtent.width);
av_log(avctx, AV_LOG_VERBOSE, " Height: from %i to %i\n",
caps->minCodedExtent.height, caps->maxCodedExtent.height);
av_log(avctx, AV_LOG_VERBOSE, " Width alignment: %i\n",
caps->pictureAccessGranularity.width);
av_log(avctx, AV_LOG_VERBOSE, " Height alignment: %i\n",
caps->pictureAccessGranularity.height);
av_log(avctx, AV_LOG_VERBOSE, " Bitstream offset alignment: %"PRIu64"\n",
caps->minBitstreamBufferOffsetAlignment);
av_log(avctx, AV_LOG_VERBOSE, " Bitstream size alignment: %"PRIu64"\n",
caps->minBitstreamBufferSizeAlignment);
av_log(avctx, AV_LOG_VERBOSE, " Maximum references: %u\n",
caps->maxDpbSlots);
av_log(avctx, AV_LOG_VERBOSE, " Maximum active references: %u\n",
caps->maxActiveReferencePictures);
av_log(avctx, AV_LOG_VERBOSE, " Codec header name: '%s' (driver), '%s' (compiled)\n",
caps->stdHeaderVersion.extensionName,
vk_desc->ext_props.extensionName);
av_log(avctx, AV_LOG_VERBOSE, " Codec header version: %i.%i.%i (driver), %i.%i.%i (compiled)\n",
CODEC_VER(caps->stdHeaderVersion.specVersion),
CODEC_VER(vk_desc->ext_props.specVersion));
av_log(avctx, AV_LOG_VERBOSE, " Decode modes:%s%s%s\n",
dec_caps->flags ? "" :
" invalid",
dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR ?
" reuse_dst_dpb" : "",
dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR ?
" dedicated_dpb" : "");
av_log(avctx, AV_LOG_VERBOSE, " Capability flags:%s%s%s\n",
caps->flags ? "" :
" none",
caps->flags & VK_VIDEO_CAPABILITY_PROTECTED_CONTENT_BIT_KHR ?
" protected" : "",
caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR ?
" separate_references" : "");
/* Check if decoding is possible with the given parameters */
if (avctx->coded_width < caps->minCodedExtent.width ||
avctx->coded_height < caps->minCodedExtent.height ||
avctx->coded_width > caps->maxCodedExtent.width ||
avctx->coded_height > caps->maxCodedExtent.height)
return AVERROR(EINVAL);
if (!(avctx->hwaccel_flags & AV_HWACCEL_FLAG_IGNORE_LEVEL) &&
avctx->level > max_level)
return AVERROR(EINVAL);
/* Some basic sanity checking */
if (!(dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR))) {
av_log(avctx, AV_LOG_ERROR, "Buggy driver signals invalid decoding mode: neither "
"VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR nor "
"VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR are set!\n");
return AVERROR_EXTERNAL;
} else if ((dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR)) ==
VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR &&
!(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) {
av_log(avctx, AV_LOG_ERROR, "Cannot initialize Vulkan decoding session, buggy driver: "
"VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR set "
"but VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR is unset!\n");
return AVERROR_EXTERNAL;
}
dec->dedicated_dpb = !!(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR);
ctx->common.layered_dpb = !dec->dedicated_dpb ? 0 :
!(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
if (dec->dedicated_dpb) {
fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
} else {
fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
VK_IMAGE_USAGE_SAMPLED_BIT;
if ((ctx->s.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
(ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
fmt_info.imageUsage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
}
/* Get the format of the images necessary */
ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
&fmt_info,
&nb_out_fmts, NULL);
if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
(!nb_out_fmts && ret == VK_SUCCESS)) {
return AVERROR(EINVAL);
} else if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
ff_vk_ret2str(ret));
return AVERROR_EXTERNAL;
}
ret_info = av_mallocz(sizeof(*ret_info)*nb_out_fmts);
if (!ret_info)
return AVERROR(ENOMEM);
for (int i = 0; i < nb_out_fmts; i++)
ret_info[i].sType = VK_STRUCTURE_TYPE_VIDEO_FORMAT_PROPERTIES_KHR;
ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
&fmt_info,
&nb_out_fmts, ret_info);
if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
(!nb_out_fmts && ret == VK_SUCCESS)) {
av_free(ret_info);
return AVERROR(EINVAL);
} else if (ret != VK_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
ff_vk_ret2str(ret));
av_free(ret_info);
return AVERROR_EXTERNAL;
}
/* Find a format to use */
*pix_fmt = best_format = AV_PIX_FMT_NONE;
*vk_fmt = best_vkfmt = VK_FORMAT_UNDEFINED;
source_format = avctx->sw_pix_fmt;
av_log(avctx, AV_LOG_DEBUG, "Choosing best pixel format for decoding from %i:\n", nb_out_fmts);
for (int i = 0; i < nb_out_fmts; i++) {
enum AVPixelFormat tmp = ff_vk_pix_fmt_from_vkfmt(ret_info[i].format);
if (tmp == AV_PIX_FMT_NONE) {
av_log(avctx, AV_LOG_WARNING, "Invalid/unknown Vulkan format %i!\n", ret_info[i].format);
continue;
}
best_format = av_find_best_pix_fmt_of_2(tmp, best_format, source_format, 0, NULL);
if (tmp == best_format)
best_vkfmt = ret_info[i].format;
av_log(avctx, AV_LOG_DEBUG, " %s%s (Vulkan ID: %i)\n",
av_get_pix_fmt_name(tmp), tmp == best_format ? "*" : "",
ret_info[i].format);
}
av_free(ret_info);
if (best_format == AV_PIX_FMT_NONE) {
av_log(avctx, AV_LOG_ERROR, "No valid/compatible pixel format found for decoding!\n");
return AVERROR(EINVAL);
} else {
av_log(avctx, AV_LOG_VERBOSE, "Chosen frame pixfmt: %s (Vulkan ID: %i)\n",
av_get_pix_fmt_name(best_format), best_vkfmt);
}
*pix_fmt = best_format;
*vk_fmt = best_vkfmt;
*dpb_dedicate = dec->dedicated_dpb;
return 0;
}
static void free_profile_data(AVHWFramesContext *hwfc)
{
av_free(hwfc->user_opaque);
}
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
{
int err, dedicated_dpb;
AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
FFVulkanDecodeProfileData *prof = NULL;
err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
if (err < 0)
return err;
frames_ctx->format = AV_PIX_FMT_VULKAN;
frames_ctx->sw_format = avctx->sw_pix_fmt;
frames_ctx->width = avctx->coded_width;
frames_ctx->height = avctx->coded_height;
if (!DECODER_IS_SDR(avctx->codec_id)) {
prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
if (!prof)
return AVERROR(ENOMEM);
err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
&frames_ctx->sw_format,
&hwfc->format[0],
prof, &dedicated_dpb);
if (err < 0) {
av_free(prof);
return err;
}
const AVPixFmtDescriptor *pdesc = av_pix_fmt_desc_get(frames_ctx->sw_format);
frames_ctx->width = FFALIGN(frames_ctx->width, 1 << pdesc->log2_chroma_w);
frames_ctx->height = FFALIGN(frames_ctx->height, 1 << pdesc->log2_chroma_h);
frames_ctx->user_opaque = prof;
frames_ctx->free = free_profile_data;
hwfc->create_pnext = &prof->profile_list;
} else {
hwfc->format[0] = VK_FORMAT_UNDEFINED;
switch (frames_ctx->sw_format) {
case AV_PIX_FMT_GBRAP16:
/* This should be more efficient for downloading and using */
frames_ctx->sw_format = AV_PIX_FMT_RGBA64;
break;
case AV_PIX_FMT_GBRP10:
/* This saves memory bandwidth when downloading */
frames_ctx->sw_format = AV_PIX_FMT_X2BGR10;
break;
case AV_PIX_FMT_RGB24:
case AV_PIX_FMT_BGR0:
/* mpv has issues with bgr0 mapping, so just remap it */
frames_ctx->sw_format = AV_PIX_FMT_RGB0;
break;
/* DPX endian mismatch remappings */
case AV_PIX_FMT_RGB48LE:
case AV_PIX_FMT_RGB48BE: frames_ctx->sw_format = AV_PIX_FMT_GBRP16; break;
case AV_PIX_FMT_RGBA64BE: frames_ctx->sw_format = AV_PIX_FMT_RGBA64; break;
case AV_PIX_FMT_GRAY16BE: frames_ctx->sw_format = AV_PIX_FMT_GRAY16; break;
/* ProRes needs to clear the input image, which is not possible on YUV formats */
case AV_PIX_FMT_YUVA422P10:
case AV_PIX_FMT_YUVA444P10:
case AV_PIX_FMT_YUVA422P12:
case AV_PIX_FMT_YUVA444P12:
hwfc->format[3] = VK_FORMAT_R16_UNORM;
av_fallthrough;
case AV_PIX_FMT_YUV422P10:
case AV_PIX_FMT_YUV444P10:
case AV_PIX_FMT_YUV422P12:
case AV_PIX_FMT_YUV444P12:
hwfc->format[0] = VK_FORMAT_R16_UNORM;
hwfc->format[1] = VK_FORMAT_R16_UNORM;
hwfc->format[2] = VK_FORMAT_R16_UNORM;
break;
default:
break;
}
}
if (hwfc->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
VK_IMAGE_USAGE_STORAGE_BIT |
VK_IMAGE_USAGE_SAMPLED_BIT;
if (prof) {
FFVulkanDecodeShared *ctx;
hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
if (!dec->dedicated_dpb)
hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
ctx = dec->shared_ctx;
if ((ctx->s.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
(ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
}
return err;
}
static void vk_decode_free_params(AVRefStructOpaque opaque, void *obj)
{
FFVulkanDecodeShared *ctx = opaque.nc;
FFVulkanFunctions *vk = &ctx->s.vkfn;
VkVideoSessionParametersKHR *par = obj;
vk->DestroyVideoSessionParametersKHR(ctx->s.hwctx->act_dev, *par,
ctx->s.hwctx->alloc);
}
int ff_vk_decode_create_params(VkVideoSessionParametersKHR **par_ref, void *logctx, FFVulkanDecodeShared *ctx,
const VkVideoSessionParametersCreateInfoKHR *session_params_create)
{
VkVideoSessionParametersKHR *par;
const FFVulkanFunctions *vk = &ctx->s.vkfn;
VkResult ret;
par = av_refstruct_alloc_ext(sizeof(*par), 0, ctx, vk_decode_free_params);
if (!par)
return AVERROR(ENOMEM);
/* Create session parameters */
ret = vk->CreateVideoSessionParametersKHR(ctx->s.hwctx->act_dev, session_params_create,
ctx->s.hwctx->alloc, par);
if (ret != VK_SUCCESS) {
av_log(logctx, AV_LOG_ERROR, "Unable to create Vulkan video session parameters: %s!\n",
ff_vk_ret2str(ret));
*par = VK_NULL_HANDLE;
av_refstruct_unref(&par);
return AVERROR_EXTERNAL;
}
*par_ref = par;
return 0;
}
int ff_vk_decode_uninit(AVCodecContext *avctx)
{
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
av_freep(&dec->hevc_headers);
av_refstruct_unref(&dec->session_params);
av_refstruct_unref(&dec->shared_ctx);
av_freep(&dec->slice_off);
return 0;
}
int ff_vk_decode_init(AVCodecContext *avctx)
{
int err;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
FFVulkanDecodeShared *ctx;
FFVulkanContext *s;
int async_depth;
const VkVideoProfileInfoKHR *profile;
const FFVulkanDecodeDescriptor *vk_desc;
VkVideoSessionCreateInfoKHR session_create = {
.sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_CREATE_INFO_KHR,
};
err = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_VULKAN);
if (err < 0)
return err;
/* Initialize contexts */
ctx = dec->shared_ctx;
s = &ctx->s;
err = ff_vk_init(s, avctx, NULL, avctx->hw_frames_ctx);
if (err < 0)
return err;
vk_desc = get_codecdesc(avctx->codec_id);
profile = get_video_profile(ctx, avctx->codec_id);
if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
return AVERROR(EINVAL);
}
/* Create queue context */
vk_desc = get_codecdesc(avctx->codec_id);
ctx->qf = ff_vk_qf_find(s, vk_desc->queue_flags, vk_desc->decode_op);
if (!ctx->qf) {
av_log(avctx, AV_LOG_ERROR, "Decoding of %s is not supported by this device\n",
avcodec_get_name(avctx->codec_id));
return err;
}
session_create.queueFamilyIndex = ctx->qf->idx;
session_create.maxCodedExtent = ctx->caps.maxCodedExtent;
session_create.maxDpbSlots = ctx->caps.maxDpbSlots;
session_create.maxActiveReferencePictures = ctx->caps.maxActiveReferencePictures;
session_create.pictureFormat = s->hwfc->format[0];
session_create.referencePictureFormat = session_create.pictureFormat;
session_create.pStdHeaderVersion = &vk_desc->ext_props;
session_create.pVideoProfile = profile;
#ifdef VK_KHR_video_maintenance2
if (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)
session_create.flags = VK_VIDEO_SESSION_CREATE_INLINE_SESSION_PARAMETERS_BIT_KHR;
#endif
/* Create the decode exec context.
* A small fixed depth is enough to keep the GPU fed, as submissions are
* serialized for hardware decoding; thread-safe hwaccels submit from
* every frame thread concurrently, and need a context per thread. */
async_depth = FF_VK_DEFAULT_EXEC_CONTEXTS;
if (ffhwaccel(avctx->hwaccel)->caps_internal & HWACCEL_CAP_THREAD_SAFE)
async_depth = FFMAX(async_depth, avctx->thread_count);
err = ff_vk_exec_pool_init(s, ctx->qf, &ctx->exec_pool,
async_depth, 0, 0, 0, profile);
if (err < 0)
goto fail;
if (!DECODER_IS_SDR(avctx->codec_id)) {
err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
if (err < 0)
goto fail;
}
/* Out-of-place decoding: create the internal, queue-exclusive DPB pool */
if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL;
void *create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
/* Reference (DPB) images use the same tiling and pNext chain as output.
* If VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR is 0, the
* driver does not support separate output and DPB with different layouts/tiling. */
void *drm_create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
if (drm_create_pnext) {
create_pnext = drm_create_pnext;
tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
av_assert2(ff_vk_find_struct(drm_create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR));
}
err = ff_vk_video_dpb_init(s, &ctx->common, s->hwfc->format[0],
VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR,
tiling, create_pnext,
s->frames->width, s->frames->height,
ctx->common.layered_dpb ?
ctx->caps.maxDpbSlots : 1);
if (err < 0)
goto fail;
if (ctx->common.layered_dpb) {
FFVulkanFunctions *vk = &ctx->s.vkfn;
FFVkExecContext *exec;
VkImageMemoryBarrier2 img_bar;
ctx->common.layered_img = av_refstruct_pool_get(ctx->common.dpb->img_pool);
if (!ctx->common.layered_img) {
err = AVERROR(ENOMEM);
goto fail;
}
ctx->common.layered_view = ctx->common.layered_img->view;
ctx->common.layered_aspect = ctx->common.layered_img->aspect;
/* Created eagerly, so transitioned eagerly; per-picture images
* transition on first use instead */
exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
err = ff_vk_exec_start(&ctx->s, exec);
if (err < 0)
goto fail;
img_bar = (VkImageMemoryBarrier2) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_NONE,
.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
.srcAccessMask = VK_ACCESS_2_NONE,
.dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = ctx->common.layered_img->img,
.subresourceRange = (VkImageSubresourceRange) {
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
.levelCount = 1,
},
};
vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) {
.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
.pImageMemoryBarriers = &img_bar,
.imageMemoryBarrierCount = 1,
});
err = ff_vk_exec_submit(&ctx->s, exec);
if (err < 0)
goto fail;
ctx->common.layered_img->layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
}
}
if (!DECODER_IS_SDR(avctx->codec_id)) {
err = decode_reset(avctx, ctx);
if (err < 0)
return err;
} else {
/* For SDR decoders, this alignment value will be 0. Since this will make
* add_slice() malfunction, set it to a sane default value. */
ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
}
const VkPhysicalDeviceDriverProperties *driver_props;
driver_props = &dec->shared_ctx->s.driver_props;
if (driver_props->driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&
driver_props->conformanceVersion.major == 1 &&
driver_props->conformanceVersion.minor == 3 &&
driver_props->conformanceVersion.subminor == 8 &&
driver_props->conformanceVersion.patch < 3)
dec->quirk_av1_offset = 1;
av_log(avctx, AV_LOG_VERBOSE, "Vulkan decoder initialization successful\n");
return 0;
fail:
ff_vk_decode_uninit(avctx);
return err;
}
View Git Blame Copy Permalink