diff --git a/.github/workflows/behavior-check.yml b/.github/workflows/behavior-check.yml index 22d8d70344..7c5ecacdcf 100644 --- a/.github/workflows/behavior-check.yml +++ b/.github/workflows/behavior-check.yml @@ -107,3 +107,52 @@ jobs: source .venv/bin/activate cd build ctest -L behavior -LE 'model|backend' --verbose --timeout 900 + + windows: + runs-on: windows-2025 + + cache-mode: write + permissions: + actions: write + contents: read + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + key: behavior-check-windows-2025 + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Python Dependencies + id: python_depends + shell: bash + run: | + python3 -m venv .venv + source .venv/Scripts/activate + pip install jinja2==3.1.6 + + - name: Build + id: cmake_build + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + cmake -B build -G "Ninja Multi-Config" ^ + -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_NATIVE=OFF ^ + -DGGML_BACKEND_DL=OFF + set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 + cmake --build build --config Release -j %NINJA_JOBS% + + - name: Test + id: cmake_test + shell: bash + run: | + source .venv/Scripts/activate + cd build + ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900 diff --git a/docs/backend/OPENCL.md b/docs/backend/OPENCL.md index 337b0c82a0..acaaedfbfc 100644 --- a/docs/backend/OPENCL.md +++ b/docs/backend/OPENCL.md @@ -72,6 +72,8 @@ However, A6x GPUs in phones are likely not supported due to the outdated driver | MXFP4 | Support | | IQ4_NL | Support | +`MUL_MAT_ID` accepts an expert id of `-1` and writes zeros to that slot. `ADD_ID` adds no bias for `-1`, including in fused MoE operations. + ## Model Preparation Since common quantizations are supported now, it is recommanded to download GGUF models directly from Huggingface. diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index f0682ce7ab..ea4e3f386d 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -324,6 +324,9 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs - `GGML_HEXAGON_NHVX=0` Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). +- `GGML_HEXAGON_VTCM_SIZE=0` + Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session. + - `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) Enables allocating host buffers for debugging. By default, host buffers are disabled. diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 5735f96f8e..85e17018c6 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -1914,213 +1914,87 @@ void ggml_compute_forward_repeat_back( } // ggml_compute_forward_concat -static void ggml_compute_forward_concat_any( - const ggml_compute_params * params, - ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; +// rows of dst copied together, so that a strided source is read along neighboring rows +static constexpr int64_t GGML_CONCAT_TILE = 32; - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - GGML_ASSERT(ggml_is_contiguous_rows(src0)); - GGML_ASSERT(ggml_is_contiguous_rows(src1)); - - int64_t o[4] = {0, 0, 0, 0}; - - if (dim == 0) { - GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); - GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); - - o[dim] = src0->ne[dim]/ggml_blck_size(src0->type); - } else { - o[dim] = src0->ne[dim]; - } - - // Region 1: copy rows from src0 - for (int i3 = 0; i3 < ne03; i3++) { - for (int i2 = ith; i2 < ne02; i2 += nth) { - for (int i1 = 0; i1 < ne01; i1++) { - const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03; - char * y = ( char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; - memcpy(y, x, ggml_row_size(src0->type, ne00)); - } - } - } - - // Region 2: copy rows from src1, offset into dst by o[] - for (int i3 = 0; i3 < ne13; i3++) { - for (int i2 = ith; i2 < ne12; i2 += nth) { - for (int i1 = 0; i1 < ne11; i1++) { - const char * x = (const char *) src1->data + i1*nb11 + i2*nb12 + i3*nb13; - char * y = ( char *) dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0; - memcpy(y, x, ggml_row_size(src1->type, ne10)); - } +// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element +template +static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) { + for (int64_t i0 = 0; i0 < n; i0++) { + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es); } } } -static void ggml_compute_forward_concat_i8( - const ggml_compute_params * params, - ggml_tensor * dst) { +// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst +static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) { + const size_t es = ggml_type_size(src->type); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const int8_t * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); - - *y = *x; - } - } - } - } -} - -static void ggml_compute_forward_concat_f16( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const ggml_fp16_t * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); - - *y = *x; - } - } - } - } -} - -static void ggml_compute_forward_concat_f32( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const float * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); - - *y = *x; - } - } + if (src->nb[0] == es) { + const size_t row_size = ggml_row_size(src->type, n); + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby, x + r*nbx, row_size); } + return; + } + + switch (es) { + case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + default: GGML_ABORT("unsupported element size"); } } +// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1 void ggml_compute_forward_concat( const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; - switch (src0->type) { - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_I16: - { - ggml_compute_forward_concat_f16(params, dst); - } break; - case GGML_TYPE_I8: - { - ggml_compute_forward_concat_i8(params, dst); - } break; - case GGML_TYPE_F32: - case GGML_TYPE_I32: - { - ggml_compute_forward_concat_f32(params, dst); - } break; - default: - { - ggml_compute_forward_concat_any(params, dst); - } + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t dim = ggml_get_op_params_i32(dst, 0); + + GGML_ASSERT(dim >= 0 && dim < 4); + GGML_ASSERT(nb0 == ggml_type_size(dst->type)); + GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0)); + GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0)); + + const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0; + + const int64_t nr = ne1*ne2*ne3; + const int64_t dr = (nr + params->nth - 1)/params->nth; + + const int64_t ir0 = dr*params->ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ) { + int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) }; + + // a tile of consecutive rows sharing i2, i3 and their source + int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE); + if (dim == 1 && i[1] < ne01) { + nt = MIN(nt, ne01 - i[1]); + } + + char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3; + + if (dim == 0 || i[dim] < src0->ne[dim]) { + ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt); + } + + if (dim == 0 || i[dim] >= src0->ne[dim]) { + i[dim] -= src0->ne[dim]; // i[0] does not address rows + ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt); + } + + ir += nt; } } @@ -4972,8 +4846,8 @@ static void ggml_compute_forward_scale_f32( const ggml_tensor * src0 = dst->src[0]; - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); float s; // scale factor @@ -4988,6 +4862,8 @@ static void ggml_compute_forward_scale_f32( const int nc = src0->ne[0]; const int nr = ggml_nrows(src0); + GGML_TENSOR_UNARY_OP_LOCALS + // rows per thread const int dr = (nr + nth - 1)/nth; @@ -4995,25 +4871,23 @@ static void ggml_compute_forward_scale_f32( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - const size_t nb01 = src0->nb[1]; + for (int i = ir0; i < ir1; i++) { + const int64_t i1 = i % ne01; + const int64_t i2 = (i / ne01) % ne02; + const int64_t i3 = i / (ne01*ne02); - const size_t nb1 = dst->nb[1]; + float * y = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); - if (b == 0.0f) { - for (int i1 = ir0; i1 < ir1; i1++) { + if (b == 0.0f) { if (dst->data != src0->data) { // src0 is same shape as dst => same indices // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy - memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float)); + memcpy(y, x, nc * sizeof(float)); } - ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s); - } - } else { - for (int i1 = ir0; i1 < ir1; i1++) { - ggml_vec_mad1_f32(nc, - (float *) ((char *) dst->data + i1*nb1), - (float *) ((char *) src0->data + i1*nb1), - s, b); + ggml_vec_scale_f32(nc, y, s); + } else { + ggml_vec_mad1_f32(nc, y, x, s, b); } } } @@ -5024,8 +4898,8 @@ static void ggml_compute_forward_scale_bf16( const ggml_tensor * src0 = dst->src[0]; - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(dst)); GGML_ASSERT(ggml_are_same_shape(src0, dst)); float s; // scale factor @@ -5040,6 +4914,8 @@ static void ggml_compute_forward_scale_bf16( const int nc = src0->ne[0]; const int nr = ggml_nrows(src0); + GGML_TENSOR_UNARY_OP_LOCALS + // rows per thread const int dr = (nr + nth - 1)/nth; @@ -5047,9 +4923,13 @@ static void ggml_compute_forward_scale_bf16( const int ir0 = dr*ith; const int ir1 = MIN(ir0 + dr, nr); - for (int i1 = ir0; i1 < ir1; i1++) { - const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]); - ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*dst->nb[1]); + for (int i = ir0; i < ir1; i++) { + const int64_t i1 = i % ne01; + const int64_t i2 = (i / ne01) % ne02; + const int64_t i3 = i / (ne01*ne02); + + const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); + ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); for (int k = 0; k < nc; k++) { y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b); } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index b795951b2f..aba21524f6 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3822,6 +3822,11 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return false; } + // TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense + if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) { + return false; + } + if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) { return false; } @@ -5972,7 +5977,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_LOG: return true; case GGML_OP_SCALE: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type; + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type && + ggml_is_contiguous_rows(op->src[0]); case GGML_OP_CLAMP: return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_ADD: diff --git a/ggml/src/ggml-cuda/scale.cu b/ggml/src/ggml-cuda/scale.cu index 5648b9261c..d53f906c53 100644 --- a/ggml/src/ggml-cuda/scale.cu +++ b/ggml/src/ggml-cuda/scale.cu @@ -3,23 +3,45 @@ #define MAX_GRIDDIM_X 0x7FFFFFFF +// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements). template -static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) { +static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k, + const uint3 ne0, const uint3 ne1, const uint3 ne2, + const uint32_t s01, const uint32_t s02, const uint32_t s03, + const uint32_t s1, const uint32_t s2, const uint32_t s3) { ggml_cuda_pdl_lc(); - int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x; - int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x; + const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; ggml_cuda_pdl_sync(); - for (int64_t i = tid; i < nelements; i += stride) { - dst[i] = ggml_cuda_cast(scale * ggml_cuda_cast(x[i]) + bias); + if (i >= k) { + return; } + + const uint2 d0 = fast_div_modulo(i, ne0); // + const uint2 d1 = fast_div_modulo(d0.x, ne1); // + const uint2 d2 = fast_div_modulo(d1.x, ne2); // + + const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03; + const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3; + + dst[i_dst] = ggml_cuda_cast(scale * ggml_cuda_cast(x[i_src]) + bias); } template -static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) { - const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE; +static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) { + const int64_t k = ggml_nelements(src0); + const size_t ts = sizeof(T); + GGML_ASSERT(k <= std::numeric_limits::max()); + + const uint3 ne0 = init_fastdiv_values(src0->ne[0]); + const uint3 ne1 = init_fastdiv_values(src0->ne[1]); + const uint3 ne2 = init_fastdiv_values(src0->ne[2]); + + const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE; const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream); - ggml_cuda_kernel_launch(scale_f32, launch_params, x, dst, scale, bias, nelements); + ggml_cuda_kernel_launch(scale_f32, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2, + src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts, + t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts); } void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -28,6 +50,7 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(dst->type == src0->type); + GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst)); float scale; float bias; @@ -35,8 +58,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); if (src0->type == GGML_TYPE_BF16) { - scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream); + scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream); } else { - scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream); + scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream); } } diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 45e6ae902a..cf2e37aa6e 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -85,6 +85,7 @@ int opt_arch = 0; // autodetect size_t opt_ndev = 1; size_t opt_nhvx = 0; // use all int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only +size_t opt_vtcm_size = 0; // VTCM size override in bytes; 0 = auto-detect size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings int opt_etm = 0; int opt_verbose = 0; @@ -2152,6 +2153,12 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; this->vtcm_size = (uint64_t)hw_vtcm_size; + if (opt_vtcm_size > 0) { + if (opt_vtcm_size > hw_vtcm_size) { + GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size); + } + this->vtcm_size = opt_vtcm_size; + } GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", this->c_name(), this->n_threads, this->n_hvx, this->n_hmx, (unsigned long long)(this->vtcm_size / (1024 * 1024))); @@ -2161,7 +2168,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? 1 : 0; - this->vtcm_size = 8 * 1024 * 1024; + this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; } } @@ -2230,7 +2237,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode); + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem, + (uint32_t) opt_power_mode); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); @@ -3391,6 +3399,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_etm = getenv("GGML_HEXAGON_ETM"); const char * str_nhvx = getenv("GGML_HEXAGON_NHVX"); const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); + const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE"); const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT"); @@ -3456,6 +3465,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { } opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx; + opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size) : opt_vtcm_size; opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split; diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idl index 303dbfb0aa..6e3390ba19 100644 --- a/ggml/src/ggml-hexagon/htp/htp_iface.idl +++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl @@ -11,7 +11,7 @@ struct htp_iface_pmu_conf { }; interface htp_iface : remote_handle64 { - AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode); + AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode); AEEResult stop(); AEEResult mmap(in uint32 fd, in uint64 size); AEEResult munmap(in uint32 fd); diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index d1a4afcd67..414c6f6617 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -325,8 +325,11 @@ static unsigned int vtcm_query_size(void) { return vtcm_size; } -static int vtcm_alloc(struct htp_context * ctx) { +static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) { unsigned int vtcm_size = vtcm_query_size(); + if (size_override) { + vtcm_size = (unsigned int) size_override; + } compute_res_attr_t attr; HAP_compute_res_attr_init(&attr); @@ -438,7 +441,7 @@ static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] = #undef HTP_POWER_PROFILE_ENTRY -AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) { +AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) { struct htp_handle * h = (struct htp_handle *) handle; if (!h) { return AEE_EBADPARM; @@ -557,7 +560,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp ctx->max_vmem = max_vmem; ctx->dsp_queue = dsp_queue; - err = vtcm_alloc(ctx); + err = vtcm_alloc(ctx, vtcm_size); if (err != AEE_SUCCESS) { FARF(ERROR, "Unable to allocate VTCM"); htp_iface_stop(handle); diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 784bb37b9a..b4b943058f 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -16,6 +16,7 @@ #include "hvx-dump.h" #include "hvx-arith.h" #include "hvx-reduce.h" +#include "hvx-copy.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -1413,7 +1414,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]); - if (eid == -1) { + if (eid < 0) { continue; } assert(eid >= 0 && eid < (int32_t) n_ids); @@ -4111,7 +4112,7 @@ static int hvx_mm_matmul_id_nx( return HTP_STATUS_OK; } -static inline void scan_expert_ids_n( +static __attribute__((noinline)) uint32_t scan_expert_ids_n( const struct htp_tensor * ids, const uint32_t n_ids, uint32_t n_as, @@ -4121,15 +4122,17 @@ static inline void scan_expert_ids_n( ) { const size_t ids_nb1 = ids->nb[1]; const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); for (uint32_t id = 0; id < n_ids; ++id) { const int32_t i02 = row_ptr[id]; - if (i02 == -1) { + if (i02 < 0) { + n_skipped++; continue; } - assert(i02 >= 0 && i02 < n_as); + assert(i02 >= 0 && i02 < (int32_t) n_as); if (matrix_rows) { matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; @@ -4137,31 +4140,10 @@ static inline void scan_expert_ids_n( counts[i02] += 1; } } + return n_skipped; } -// the matmuls only write the rows that an expert owns, so the rows of the skipped slots are zeroed here -static inline void zero_skipped_dst_rows( - const struct htp_tensor * ids, - uint32_t n_ids, - const struct htp_tensor * dst, - uint32_t ne0, - size_t nb1, - size_t nb2 -) { - const uint8_t * ids_data = (const uint8_t *) ids->data; - - for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { - for (uint32_t id = 0; id < n_ids; ++id) { - const int32_t i02 = *(const int32_t *) (ids_data + iid1 * ids->nb[1] + id * ids->nb[0]); - if (i02 != -1) { - continue; - } - memset((uint8_t *) (dst->data + id * nb1 + iid1 * nb2), 0, ne0 * sizeof(float)); - } - } -} - -static inline void scan_expert_ids( +static __attribute__((noinline)) uint32_t scan_expert_ids( const struct htp_tensor * ids, uint32_t n_ids, uint32_t n_as, @@ -4173,23 +4155,25 @@ static inline void scan_expert_ids( if (ids_nb0 == 4) { switch (n_ids) { - case 8: scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); break; - case 4: scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); break; - case 2: scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); break; - default: scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); break; + case 8: return scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); + case 4: return scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); + case 2: return scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); + default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); } } else { // Strided fallback const size_t ids_nb1 = ids->nb[1]; const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); for (uint32_t id = 0; id < n_ids; ++id) { const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0); - if (i02 == -1) { + if (i02 < 0) { + n_skipped++; continue; } - assert(i02 >= 0 && i02 < n_as); + assert(i02 >= 0 && i02 < (int32_t) n_as); if (matrix_rows) { matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; @@ -4197,9 +4181,124 @@ static inline void scan_expert_ids( counts[i02] += 1; } } + return n_skipped; } } +static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) { + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + uint32_t n_skipped = 0; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0); + if (i02 < 0) { + n_skipped++; + } + } + return n_skipped; +} + +// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns. +static void zero_skipped_dst_rows( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + const struct htp_tensor * dst +) { + const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + uint32_t col_start = 0; + uint32_t col_count = dst->ne[0]; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, elem_size); + const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + col_start = range.start; + col_count = range.count; + } + + if (col_count == 0) { + return; + } + + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + const size_t ids_nb1 = ids->nb[1]; + const size_t dst_nb1 = dst->nb[1]; + const size_t dst_nb2 = dst->nb[2]; + const uint32_t col_offset_bytes = col_start * elem_size; + + for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + const uint8_t * ids_row = ids_data + iid1 * ids_nb1; + uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0); + if (i02 < 0) { + hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size); + } + } + } +} + +static int htp_mmid_prepare_mapping( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + uint32_t n_as, + uint32_t act_nrows, + uint8_t ** out_mapping_buf, + uint32_t ** out_matrix_row_counts, + struct mmid_row_mapping ** out_matrix_rows, + uint32_t * out_mapping_stride, + uint32_t * out_n_skipped +) { + uint8_t * mapping_buf = octx->ctx->ddr_spad_base; + uint32_t mapping_stride = 1; + uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; + struct mmid_row_mapping * matrix_rows = NULL; + uint32_t n_skipped = 0; + + if (act_nrows > 1) { + const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); + assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); + + hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); + + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); + n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); + + uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); + mapping_stride = max_count > 0 ? max_count : 1; + + size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping); + const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size; + + if (total_map_size > octx->ctx->ddr_spad_size) { + mapping_buf = memalign(128, total_map_size); + if (!mapping_buf) { + return HTP_STATUS_INTERNAL_ERR; + } + } + + matrix_row_counts = (uint32_t *) mapping_buf; + matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); + + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); + scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); + } else { + n_skipped = count_skipped_ids_single(ids, n_ids); + } + + *out_mapping_buf = mapping_buf; + *out_matrix_row_counts = matrix_row_counts; + *out_matrix_rows = matrix_rows; + *out_mapping_stride = mapping_stride; + *out_n_skipped = n_skipped; + + return HTP_STATUS_OK; +} + int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; @@ -4235,41 +4334,22 @@ int op_matmul_id(struct htp_ops_context * octx) { const int n_ids = ids->ne[0]; // n_expert_used const int n_as = ne02; // n_expert - uint8_t * mapping_buf = octx->ctx->ddr_spad_base; - uint32_t mapping_stride = 1; - uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; - if (act_nrows > 1) { - const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); - assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); - - hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); - - memset(matrix_row_counts, 0, matrix_row_counts_size); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); - - uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); - mapping_stride = max_count > 0 ? max_count : 1; - - size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping); - const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size; - - if (total_map_size > octx->ctx->ddr_spad_size) { - mapping_buf = memalign(128, total_map_size); - if (!mapping_buf) { - return HTP_STATUS_INTERNAL_ERR; - } - } - - matrix_row_counts = (uint32_t *) mapping_buf; - matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); - - memset(matrix_row_counts, 0, n_as * sizeof(uint32_t)); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; } - zero_skipped_dst_rows(ids, n_ids, dst, ne0, nb1, nb2); + if (n_skipped > 0) { + zero_skipped_dst_rows(octx, ids, n_ids, dst); + } mmctx->matrix_row_counts = matrix_row_counts; mmctx->matrix_rows = matrix_rows; @@ -4360,38 +4440,26 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { const int n_ids = ids->ne[0]; const int n_as = src0->ne[2]; - uint8_t * mapping_buf = octx->ctx->ddr_spad_base; - uint32_t mapping_stride = 1; - uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; - if (act_nrows > 1) { - const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); - assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; + } - hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); - - memset(matrix_row_counts, 0, matrix_row_counts_size); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); - - uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); - mapping_stride = max_count > 0 ? max_count : 1; - - size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping); - const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size; - - if (total_map_size > octx->ctx->ddr_spad_size) { - mapping_buf = memalign(128, total_map_size); - if (!mapping_buf) { - return HTP_STATUS_INTERNAL_ERR; + if (n_skipped > 0) { + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict d = octx->dsts[p]; + if (d) { + zero_skipped_dst_rows(octx, ids, n_ids, d); } } - - matrix_row_counts = (uint32_t *) mapping_buf; - matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); - - memset(matrix_row_counts, 0, n_as * sizeof(uint32_t)); - scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); } mmctx->matrix_row_counts = matrix_row_counts; diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index d1bce8bad8..31ffe05854 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1555,6 +1555,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te switch (op->op) { case GGML_OP_SCALE: + return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16); case GGML_OP_FILL: case GGML_OP_CLAMP: case GGML_OP_SQR: diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metal index e50a648639..8c2c18ebd6 100644 --- a/ggml/src/ggml-metal/kernels/unary.metal +++ b/ggml/src/ggml-metal/kernels/unary.metal @@ -3,6 +3,24 @@ constant short FC_unary_op [[function_constant(FC_UNARY + 0)]]; constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]]; +template +inline constexpr T ggml_metal_unary_cast(U x) { + return (T) x; +} + +#if defined(GGML_METAL_HAS_BF16) +// MSL has no C-style cast from float vectors to bfloat vectors +template <> +inline constexpr bfloat4 ggml_metal_unary_cast(float4 x) { + return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3])); +} + +template <> +inline constexpr bfloat4 ggml_metal_unary_cast(float x) { + return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x)); +} +#endif + template kernel void kernel_unary_impl( constant ggml_metal_kargs_unary & args, @@ -48,79 +66,79 @@ kernel void kernel_unary_impl( const TC x = (TC) src0_ptr[i0]; if (FC_OP == OP_UNARY_NUM_SCALE) { - dst_ptr[i0] = (T) (args.scale * x + args.bias); + dst_ptr[i0] = ggml_metal_unary_cast(args.scale * x + args.bias); } if (FC_OP == OP_UNARY_NUM_FILL) { - dst_ptr[i0] = (T) args.val; + dst_ptr[i0] = ggml_metal_unary_cast(args.val); } if (FC_OP == OP_UNARY_NUM_CLAMP) { - dst_ptr[i0] = (T) clamp(x, args.min, args.max); + dst_ptr[i0] = ggml_metal_unary_cast(clamp(x, args.min, args.max)); } if (FC_OP == OP_UNARY_NUM_SQR) { - dst_ptr[i0] = (T) (x * x); + dst_ptr[i0] = ggml_metal_unary_cast(x * x); } if (FC_OP == OP_UNARY_NUM_SQRT) { - dst_ptr[i0] = (T) sqrt(x); + dst_ptr[i0] = ggml_metal_unary_cast(sqrt(x)); } if (FC_OP == OP_UNARY_NUM_SIN) { - dst_ptr[i0] = (T) sin(x); + dst_ptr[i0] = ggml_metal_unary_cast(sin(x)); } if (FC_OP == OP_UNARY_NUM_COS) { - dst_ptr[i0] = (T) cos(x); + dst_ptr[i0] = ggml_metal_unary_cast(cos(x)); } if (FC_OP == OP_UNARY_NUM_LOG) { - dst_ptr[i0] = (T) log(x); + dst_ptr[i0] = ggml_metal_unary_cast(log(x)); } if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) { - dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); + dst_ptr[i0] = ggml_metal_unary_cast(TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); } if (FC_OP == OP_UNARY_NUM_TANH) { - dst_ptr[i0] = (T) precise::tanh(x); + dst_ptr[i0] = ggml_metal_unary_cast(precise::tanh(x)); } if (FC_OP == OP_UNARY_NUM_RELU) { - dst_ptr[i0] = (T) fmax(0, x); + dst_ptr[i0] = ggml_metal_unary_cast(fmax(0, x)); } if (FC_OP == OP_UNARY_NUM_SIGMOID) { - dst_ptr[i0] = (T) (1 / (1 + exp(-x))); + dst_ptr[i0] = ggml_metal_unary_cast(1 / (1 + exp(-x))); } if (FC_OP == OP_UNARY_NUM_GELU) { - dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); + dst_ptr[i0] = ggml_metal_unary_cast(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); } if (FC_OP == OP_UNARY_NUM_GELU_ERF) { - dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x))); + dst_ptr[i0] = ggml_metal_unary_cast(0.5*x*(1 + erf_approx(SQRT_2_INV*x))); } if (FC_OP == OP_UNARY_NUM_GELU_QUICK) { - dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x)))); + dst_ptr[i0] = ggml_metal_unary_cast(x * (1/(1 + exp(GELU_QUICK_COEF*x)))); } if (FC_OP == OP_UNARY_NUM_SILU) { - dst_ptr[i0] = (T) (x / (1 + exp(-x))); + dst_ptr[i0] = ggml_metal_unary_cast(x / (1 + exp(-x))); } if (FC_OP == OP_UNARY_NUM_ELU) { - dst_ptr[i0] = (T) elu_approx(x); + dst_ptr[i0] = ggml_metal_unary_cast(elu_approx(x)); } if (FC_OP == OP_UNARY_NUM_NEG) { - dst_ptr[i0] = (T) -x; + dst_ptr[i0] = ggml_metal_unary_cast(-x); } if (FC_OP == OP_UNARY_NUM_ABS) { - dst_ptr[i0] = (T) fabs(x); + dst_ptr[i0] = ggml_metal_unary_cast(fabs(x)); } if (FC_OP == OP_UNARY_NUM_SGN) { @@ -132,40 +150,40 @@ kernel void kernel_unary_impl( } if (FC_OP == OP_UNARY_NUM_HARDSWISH) { - dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5))); + dst_ptr[i0] = ggml_metal_unary_cast(x * fmax(0, fmin(1, x/6 + 0.5))); } if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) { - dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5)); + dst_ptr[i0] = ggml_metal_unary_cast(fmax(0, fmin(1, x/6 + 0.5))); } if (FC_OP == OP_UNARY_NUM_EXP) { - dst_ptr[i0] = (T) exp(x); + dst_ptr[i0] = ggml_metal_unary_cast(exp(x)); } if (FC_OP == OP_UNARY_NUM_SOFTPLUS) { - dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20); + dst_ptr[i0] = ggml_metal_unary_cast(select(log(1 + exp(x)), x, x > 20)); } if (FC_OP == OP_UNARY_NUM_EXPM1) { // TODO: precise implementation - dst_ptr[i0] = (T) (exp(x) - 1); + dst_ptr[i0] = ggml_metal_unary_cast(exp(x) - 1); } if (FC_OP == OP_UNARY_NUM_FLOOR) { - dst_ptr[i0] = (T) floor(x); + dst_ptr[i0] = ggml_metal_unary_cast(floor(x)); } if (FC_OP == OP_UNARY_NUM_CEIL) { - dst_ptr[i0] = (T) ceil(x); + dst_ptr[i0] = ggml_metal_unary_cast(ceil(x)); } if (FC_OP == OP_UNARY_NUM_ROUND) { - dst_ptr[i0] = (T) round(x); + dst_ptr[i0] = ggml_metal_unary_cast(round(x)); } if (FC_OP == OP_UNARY_NUM_TRUNC) { - dst_ptr[i0] = (T) trunc(x); + dst_ptr[i0] = ggml_metal_unary_cast(trunc(x)); } if (FC_OP == OP_UNARY_NUM_XIELU) { @@ -174,7 +192,7 @@ kernel void kernel_unary_impl( const TC clamped = fmin(xi, TC(args.val)); const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi; const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi; - dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg); + dst_ptr[i0] = ggml_metal_unary_cast(gate * y_pos + (TC(1.0f) - gate) * y_neg); } } @@ -188,6 +206,10 @@ template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_un template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl; template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl; template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_unary_bf16_bf16")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl; +#endif kernel void kernel_silu_back_f32( constant ggml_metal_kargs_silu_back & args, diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 57620e560e..1963c69434 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -104,7 +104,13 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_q2_k_q8_1_dp4a gemv_noshuffle_q2_k_f32 gemm_noshuffle_q2_k_f32 + gemm_noshuffle_q3_k_f32 + gemm_noshuffle_q3_k_q8_1_dp4a + gemv_noshuffle_q3_k_f32 mul_mv_q2_k_f32 + mul_mv_q3_k_f32 + mul_mv_q3_k_f32_flat + mul_mm_q3_k_f32_l4_lm mul_mv_q2_k_f32_flat mul_mm_q2_k_f32_l4_lm mul_mv_q4_k_f32 @@ -128,6 +134,7 @@ set(GGML_OPENCL_KERNELS mul_mv_id_q8_0_f32_flat mul_mv_id_mxfp4_f32 mul_mv_id_mxfp4_f32_flat + mul_mat_id_utils gemm_moe_q4_0_f32_ns gemm_moe_q4_0_q8_1_dp4a gemv_moe_q4_0_f32_ns diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 65a5e66214..67248264e1 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -975,6 +975,17 @@ struct ggml_backend_opencl_context { int q2_k_mv_rows = 4; int q2_k_mv_nsg = 4; cl_kernel kernel_mul_mv_q2_K_f32; + cl_kernel kernel_convert_block_q3_K, kernel_restore_block_q3_K; + cl_kernel kernel_mul_mv_q3_K_f32_flat; + cl_kernel kernel_mul_mm_q3_k_f32_l4_lm; + cl_kernel kernel_convert_block_q3_k_noshuffle, kernel_restore_block_q3_k_noshuffle; + cl_kernel kernel_gemv_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = nullptr; + int q3_k_mv_rows = 2; + int q3_k_mv_nsg = 8; + cl_kernel kernel_mul_mv_q3_K_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -1542,6 +1553,23 @@ static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * b return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); } +static std::string ggml_cl_q3_k_mm_options(const ggml_backend_opencl_context * backend_ctx) { + const bool x2 = backend_ctx->gpu_family == ADRENO && + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + int bk = x2 ? 16 : 32; + bool half = x2; + if (const char * env = getenv("GGML_OPENCL_LM_BK")) { + const int value = atoi(env); + if (value == 16 || value == 32) { + bk = value; + } + } + if (const char * env = getenv("GGML_OPENCL_LM_HALF")) { + half = atoi(env) != 0; + } + return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); +} + static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { // compiler options for general kernels auto opencl_c_std = @@ -1815,6 +1843,21 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mat_id_utils + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mat_id_utils.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mat_id_utils.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // moe_add_id_glu { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1992,6 +2035,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err)); @@ -2398,6 +2445,53 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mv_q3_k_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q3_k_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q3_k_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_q3_k_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q3_k_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q3_k_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q3_k_mm_options(backend_ctx)); + CL_CHECK((backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q3_k_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q3_k_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4206,6 +4300,80 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q3_k_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q3_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) { + const int rows = atoi(env); + if (rows == 1 || rows == 2 || rows == 4) { + backend_ctx->q3_k_mv_rows = rows; + } + } + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->q3_k_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) + + " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel; + break; + } + CL_CHECK(clReleaseKernel(kernel)); + GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1); + backend_ctx->q3_k_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + // gemm_noshuffle_q2_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -5737,7 +5905,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); - CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); @@ -8245,6 +8412,27 @@ struct ggml_tensor_extra_cl_q2_K { } }; +struct ggml_tensor_extra_cl_q3_K { + bool noshuffle = false; + cl_mem qs = nullptr; // quantized values + cl_mem hm = nullptr; // high bits + cl_mem sc = nullptr; // packed 6-bit scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_hm = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_q3_K() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (hm != nullptr) { CL_CHECK(clReleaseMemObject(hm)); hm = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_hm = size_sc = size_d = 0; + } +}; + struct ggml_tensor_extra_cl_q6_K { // Lower 4 bits of quantized weights. cl_mem ql = nullptr; @@ -9345,6 +9533,24 @@ static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backe #endif } +static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q3_K_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q3_K && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. @@ -9868,6 +10074,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } else if (op->src[0]->type == GGML_TYPE_Q4_0) { // Non-contig src0 routes through on-device dequant-to-f16. return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q3_K) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q2_K) { const ggml_tensor * weight = op->src[0]->view_src; if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) { @@ -10254,12 +10466,18 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) { delete e; } + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) { + delete e; + } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { delete e; } + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + delete e; + } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } @@ -10448,6 +10666,21 @@ struct ggml_backend_opencl_buffer_context { return extra; } + ggml_tensor_extra_cl_q3_K * ggml_opencl_alloc_temp_tensor_extra_q3_K() { + ggml_tensor_extra_cl_q3_K * extra; + if (temp_tensor_extras_q3_K.empty()) { + extra = new ggml_tensor_extra_cl_q3_K(); + } else { + extra = temp_tensor_extras_q3_K.back(); + temp_tensor_extras_q3_K.pop_back(); + } + + temp_tensor_extras_q3_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { ggml_tensor_extra_cl_q6_K * extra; if (temp_tensor_extras_q6_K.empty()) { @@ -10529,6 +10762,11 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_q2_K_in_use.clear(); + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + temp_tensor_extras_q3_K.push_back(e); + } + temp_tensor_extras_q3_K_in_use.clear(); + q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); } @@ -10561,7 +10799,9 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_q5_K; std::vector temp_tensor_extras_q5_K_in_use; std::vector temp_tensor_extras_q2_K; + std::vector temp_tensor_extras_q3_K; std::vector temp_tensor_extras_q2_K_in_use; + std::vector temp_tensor_extras_q3_K_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; @@ -10818,6 +11058,91 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, return; } + if (tensor->type == GGML_TYPE_Q3_K) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q3_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q3_K(); + extra->noshuffle = ggml_cl_q3_k_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * (blck/4); + const size_t size_hm = n_blk * (blck/8); + const size_t size_sc = n_blk * 12; + const size_t size_d = n_blk * sizeof(ggml_fp16_t); + GGML_ASSERT(size_qs + size_hm + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment); + region.size = size_hm; + CL_CHECK((extra->hm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_hm, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q3_k_noshuffle + : backend_ctx->kernel_convert_block_q3_K; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->hm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_hm = size_hm; + extra->size_sc = size_sc; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); @@ -10901,7 +11226,16 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, // conversion kernel. if (tensor->type == GGML_TYPE_Q4_0) { // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + } return; } // Tensors should have been preallocated, therefore they should @@ -11605,7 +11939,16 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, } if (tensor->type == GGML_TYPE_Q8_0) { // Views share the parent's buffer; parent owns SoA conversion. + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q8_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + } return; } @@ -12440,7 +12783,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(extra); cl_ulong n_elements = size / sizeof(ggml_fp16_t); - cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t); + cl_ulong off_dst = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, @@ -12468,8 +12811,9 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); + // a view reuses its parent's extra, whose offset does not include view_offs CL_CHECK(clEnqueueWriteBuffer( - queue, extra->data_device, CL_TRUE, extra->offset + offset, + queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, size, data, 0, NULL, NULL)); GGML_UNUSED(buffer); @@ -12534,6 +12878,56 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } + if (tensor->type == GGML_TYPE_Q3_K) { + ggml_tensor_extra_cl_q3_K * extra = (ggml_tensor_extra_cl_q3_K *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, hm = extra->hm, sc = extra->sc, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_hm, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_hm.allocate(context, extra->size_hm); + buf_sc.allocate(context, extra->size_sc); + buf_d.allocate(context, extra->size_d); + qs = buf_qs.buffer; + hm = buf_hm.buffer; + sc = buf_sc.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle + : backend_ctx->kernel_restore_block_q3_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; @@ -24905,6 +25299,65 @@ static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor return true; } +static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 16 ? 8 : 32; + cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q3_K_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; @@ -25004,6 +25457,108 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } + +static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows; + const size_t nsg = backend_ctx->q3_k_mv_nsg; + size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} #endif static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -25075,6 +25630,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; + ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; @@ -25476,6 +26032,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } + // q3_k x fp32 + if (src0t == GGML_TYPE_Q3_K && src1t == GGML_TYPE_F32 && extra0_q3_K->noshuffle) { + ggml_cl_mul_mat_q3_k_f32_adreno(backend, src0, src1, dst); + return; + } + // q4_k x fp32 if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); @@ -26026,6 +26588,51 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } + case GGML_TYPE_Q3_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } case GGML_TYPE_Q4_K: { if (ne11 < 32) { break; @@ -27005,7 +27612,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q3_K: + case GGML_TYPE_Q3_K: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 2; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 2; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q3_K_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q3_K_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } case GGML_TYPE_Q4_K: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; @@ -27313,7 +27983,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size @@ -27329,8 +27999,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q3_K) { - GGML_ASSERT(false && "not implemented"); } else if (src0t == GGML_TYPE_Q5_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; @@ -27630,35 +28298,21 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // the matmul kernels only write the rows an expert owns, so zero the skipped rows here { - cl_int err; - - cl_buffer_region region; - region.origin = offset2; - region.size = nb21 * ne21; - cl_mem buf_ids = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - - region.origin = offsetd; - region.size = (size_t) ne0 * ne1 * ne2 * sizeof(float); - cl_mem buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - - const cl_uint ids_stride = nb21 / sizeof(int); - + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst; - CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &buf_ids)); - CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_mem), &buf_dst)); - CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_uint), &ne21)); - CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_uint), &ne20)); - CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_uint), &ids_stride)); - CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_uint), &ne0)); + CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel_zero, 6, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel_zero, 7, sizeof(int), &ne0)); size_t zero_local[3] = {1, 1, 64}; - size_t zero_global[3] = {(size_t) ne21, (size_t) ne20, 64}; + size_t zero_global[3] = {(size_t) ne20, (size_t) ne21, 64}; backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst); - - CL_CHECK(clReleaseMemObject(buf_ids)); - CL_CHECK(clReleaseMemObject(buf_dst)); } int sgs = 32; // subgroup size diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index f3309a2989..fb21e8eba2 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -2773,3 +2773,156 @@ kernel void kernel_restore_block_q2_k_noshuffle( } } } + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +kernel void kernel_convert_block_q3_K( + global struct block_q3_K * src0, + global uchar * dst_qs, + global uchar * dst_hm, + global uchar * dst_sc, + global half * dst_d, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = src0 + i; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = b->qs[j]; + } + for (int j = 0; j < 32; ++j) { + dst_hm[32*i + j] = b->hmask[j]; + } + for (int j = 0; j < 12; ++j) { + dst_sc[12*i + j] = b->scales[j]; + } + dst_d[i] = b->d; +} + +kernel void kernel_restore_block_q3_K( + global uchar * src_qs, + global uchar * src_hm, + global uchar * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = dst + i; + for (int j = 0; j < 64; ++j) { + b->qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 32; ++j) { + b->hmask[j] = src_hm[32*i + j]; + } + for (int j = 0; j < 12; ++j) { + b->scales[j] = src_sc[12*i + j]; + } + b->d = src_d[i]; +} + +kernel void kernel_convert_block_q3_k_noshuffle( + global struct block_q3_K * src0, + global uchar * dst_qs, // QK_K/4 per block + global uchar * dst_hm, // QK_K/8 per block + global uint * dst_sc, // 3 per block + global half * dst_d, // 1 + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) src0 + i; + + dst_d[i] = b->d; + + for (int w = 0; w < 3; ++w) { + global const uchar * p = b->scales + 4*w; + dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24); + } + + for (int g = 0; g < QK_K/4; ++g) { + uint pk = 0; + for (int t = 0; t < 4; ++t) { + const int e = 4*g + t; + const int sb = e >> 5; + const int ee = e & 31; + pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t)); + } + dst_qs[(QK_K/4) * i + g] = (uchar)pk; + } + + // two groups per hmask byte, so this loop owns the byte and cannot race + for (int t = 0; t < QK_K/8; ++t) { + uint v = 0; + for (int s = 0; s < 2; ++s) { + const int g = 2*t + s; + for (int u = 0; u < 4; ++u) { + const int e = 4*g + u; + const int sb = e >> 5; + const int ee = e & 31; + v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u)); + } + } + dst_hm[(QK_K/8) * i + t] = (uchar)v; + } +} + +kernel void kernel_restore_block_q3_k_noshuffle( + global uchar * src_qs, + global uchar * src_hm, + global uint * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) dst + i; + + b->d = src_d[i]; + + for (int w = 0; w < 3; ++w) { + const uint x = src_sc[3*i + w]; + global uchar * p = b->scales + 4*w; + p[0] = (uchar)( x & 0xFF); + p[1] = (uchar)((x >> 8) & 0xFF); + p[2] = (uchar)((x >> 16) & 0xFF); + p[3] = (uchar)((x >> 24) & 0xFF); + } + + for (int n2 = 0; n2 < 2; ++n2) { + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int j = 0; j < 4; ++j) { + const int sb = 4*n2 + j; + const int e = 32*sb + ee; + v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j); + } + b->qs[32*n2 + ee] = (uchar)v; + } + } + + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int sb = 0; sb < 8; ++sb) { + const int e = 32*sb + ee; + const int g = e >> 2; + v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb; + } + b->hmask[ee] = (uchar)v; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl new file mode 100644 index 0000000000..2851146787 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const int is = (i / 16) & 15; + const int word = is >> 2; + const int shift = 8 * (is & 3); + const int sc_base = row + 3 * (i / 256) * m; + const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1)); + const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word); + const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m); + const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3)); + const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl new file mode 100644 index 0000000000..04fcca0b33 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl @@ -0,0 +1,160 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline uint q3_k_pack(uint pk, uint hb) { + int v0 = (int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u); + int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u); + int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u); + int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u); + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +#define KQ_STAGE(p) vload4(0, (p)) + +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_hm, + __global const uint * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; // super-block along K + const uint sb = sub & 7u; + + const uint scb = rrow + (3u * ib) * (uint)m; + const uint sc0 = src0_sc[scb + 0u * (uint)m]; + const uint sc1 = src0_sc[scb + 1u * (uint)m]; + const uint sc2 = src0_sc[scb + 2u * (uint)m]; + + const float d_w = (float)src0_d[rrow + ib * (uint)m]; + const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u); + const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u); + + const uint qsb = rrow + (step >> 2) * (uint)m; + const uint hmb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m]; + const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m]; + const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m]; + const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m]; + qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m], h0 & 0xFu); + qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu); + qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m], h1 & 0xFu); + qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu); + qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m], h2 & 0xFu); + qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu); + qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m], h3 & 0xFu); + qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1]))) + rf.s0 = Q3_K_COL(b+0); rf.s1 = Q3_K_COL(b+1); + rf.s2 = Q3_K_COL(b+2); rf.s3 = Q3_K_COL(b+3); +#undef Q3_K_COL + acc[g] += LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl new file mode 100644 index 0000000000..f44354154c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl @@ -0,0 +1,249 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef Q3_K_MV_NSG +#define Q3_K_MV_NSG 8 +#endif + +#ifndef Q3_K_MV_R +#define Q3_K_MV_R 2 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline float4 q3_k_vals(uint pk, uint hb) { + float4 v; + v.s0 = (float)((int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u)); + v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u)); + v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u)); + v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u)); + return v; +} + +kernel void kernel_gemv_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, // K + int ne01, // M + int ne10, // activation row stride + int ne0 // dst row stride +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#if Q3_K_MV_R == 4 + const uint mq = m >> 2; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 2; + + float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f; + + if (j < mq) { + global const uint * qsw = (global const uint *)src0_qs; + global const uint * hmw = (global const uint *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half4 dh = vload4(j + ib * mq, src0_d); + const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc); + const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc); + const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc); + + float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mq; + const uint hmb = j + (grp >> 1) * mq; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h); + const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = qsw[qsb + gg * mq]; // four rows, one load + const uint hmv = hmw[hmb + (gg >> 1) * mq]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); // shared by all four + a0 += dot(yv, q3_k_vals((qsv ) & 0xFFu, ((hmv ) >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu)); + a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + acc2 += (float)l2 * a2; + acc3 += (float)l3 * a3; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + sumf2 += (float)dh.s2 * acc2; + sumf3 += (float)dh.s3 * acc3; + } + } +#elif Q3_K_MV_R == 2 + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * hmu = (global const ushort *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc); + const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc); + const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc); + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mh; + const uint hmb = j + (grp >> 1) * mh; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)qsu[qsb + gg * mh]; + const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); + a0 += dot(yv, q3_k_vals( qsv & 0xFFu, ( hmv >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + } + } +#else + const uint row = get_group_id(0) * 64u + lid; + + float sumf = 0.f; + + if (row < m) { + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const float d = (float)src0_d[row + ib * m]; + const uint s0 = src0_sc[row + (3u * ib + 0u) * m]; + const uint s1 = src0_sc[row + (3u * ib + 1u) * m]; + const uint s2 = src0_sc[row + (3u * ib + 2u) * m]; + + float acc = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = row + grp * m; + const uint hmb = row + (grp >> 1) * m; + + for (uint h = 0; h < 2u; ++h) { + const int l = q3_k_scale(s0, s1, s2, 2u*sb + h); + float a = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)src0_qs[qsb + gg * m]; + const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m]; + const float4 yv = vload4(grp + gg, y); + a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu)); + } + acc += (float)l * a; + } + } + sumf += d * acc; + } + } +#endif + +#if Q3_K_MV_NSG > 1 +#if Q3_K_MV_R == 4 + __local float4 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float4 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + sumf2 += p.s2; + sumf3 += p.s3; + } +#elif Q3_K_MV_R == 2 + __local float2 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#else + __local float part[Q3_K_MV_NSG][64]; + part[sgi][lid] = sumf; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + sumf += part[s][lid]; + } +#endif +#endif + +#if Q3_K_MV_R == 4 + if (j < mq) { + vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row); + } +#elif Q3_K_MV_R == 2 + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#else + if (row < m) { + dst[(ulong)col * (uint)ne0 + row] = sumf; + } +#endif +} diff --git a/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl index 6a8e4fb17f..df9919ec10 100644 --- a/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +++ b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl @@ -57,13 +57,17 @@ kernel void kernel_add_id_add_id_swiglu_oai( global const float * g_row = (global const float *)(src_g + i1*nb01_g + i2*nb02_g); global const float * u_row = (global const float *)(src_u + i1*nb01_u + i2*nb02_u); - global const float * gb_row = (global const float *)(src_gb + i11*nb11_g); - global const float * ub_row = (global const float *)(src_ub + i11*nb11_u); global float * d_row = (global float *)(dst + i1*nbd1 + i2*nbd2); for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { - float x0 = g_row[i0] + gb_row[i0]; - float x1 = u_row[i0] + ub_row[i0]; + float x0 = g_row[i0]; + float x1 = u_row[i0]; + if (i11 != -1) { + global const float * gb_row = (global const float *)(src_gb + i11*nb11_g); + global const float * ub_row = (global const float *)(src_ub + i11*nb11_u); + x0 += gb_row[i0]; + x1 += ub_row[i0]; + } x0 = min(x0, limit); x1 = max(min(x1, limit), -limit); diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.cl index acd08dbe66..cae7a7f2a9 100644 --- a/ggml/src/ggml-opencl/kernels/moe_combine.cl +++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl @@ -42,8 +42,11 @@ __kernel void kernel_moe_combine_bias_f32( float4 acc = (float4)(0.0f); for (int e = 0; e < k; ++e) { const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int))); - __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u; - const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow); + float4 v = vload4(0, E + (uint)e*e1); + if (i11 != -1) { + __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u; + v += vload4(0, Brow); + } acc = mad(v, (float4)(W[(uint)e*w1]), acc); } diff --git a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl index 5ea31afb8e..e8ac31af22 100644 --- a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +++ b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl @@ -80,32 +80,6 @@ __kernel void kernel_moe_scatter( emap[tile_idx] = val; } -// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here -__kernel void kernel_moe_zero_dst( - __global const int * input, - __global float * dst, - uint N, - uint topK, - uint n_experts, - uint ne0 -) { - uint n = get_global_id(0); - uint k = get_global_id(1); - - if (n >= N || k >= topK) { - return; - } - - if (input[n * n_experts + k] != -1) { - return; - } - - __global float * dst_row = dst + ((ulong)n * topK + k) * ne0; - for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) { - dst_row[i] = 0.0f; - } -} - // Deterministic replacement for kernel_moe_scatter. // // kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]), diff --git a/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl b/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl new file mode 100644 index 0000000000..79ec211632 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl @@ -0,0 +1,22 @@ +kernel void kernel_moe_zero_dst( + global const char * ids, + ulong offset_ids, + ulong nb_ids1, + global char * dst, + ulong offset_dst, + ulong nb1, + ulong nb2, + int ne0 +) { + const size_t i1 = get_group_id(0); + const size_t i2 = get_group_id(1); + const int expert = *((global const int *)(ids + offset_ids + i1*sizeof(int) + i2*nb_ids1)); + if (expert != -1) { + return; + } + + global float * row = (global float *)(dst + offset_dst + i1*nb1 + i2*nb2); + for (size_t i0 = get_local_id(2); i0 < (size_t)ne0; i0 += get_local_size(2)) { + row[i0] = 0.0f; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl new file mode 100644 index 0000000000..da3997417e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl @@ -0,0 +1,219 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#ifndef BK +#define BK 32 +#endif +#ifdef INTEL_GPU +#define TM 8 +#else +#define TM 4 +#endif +#define TN 8 + +#ifndef LM_HALF +#define LM_HALF 0 +#endif +#if LM_HALF +typedef half lm_st; +#define LM_LD4(p, i) convert_float4(vload4((i), (p))) +#else +typedef float lm_st; +#define LM_LD4(p, i) vload4((i), (p)) +#endif + +kernel void kernel_mul_mm_q3_k_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local lm_st buf_a[BM * BK]; + local lm_st buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one q3_K super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float4 sums4[(TM/4) * TN]; + float4 cache_a4[TM/4]; + + for (int i = 0; i < (TM/4) * TN; i++) { + sums4[i] = (float4)(0.0f); + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_q3_K); + int e = idx % QK_K; + + // Only qs advances in the second half of a super-block. + int n = e >> 7; + int rem = e & 127; + int j = rem >> 5; + int s = (rem >> 4) & 1; + int l16 = rem & 15; + int is = 8*n + 2*j + s; + int shift = 2*j; + uchar m = (uchar)(1u << (4*n + j)); + + global ushort * a = (global ushort *)(src0_sc + 12*ib); + uint a0 = (uint)a[0] | ((uint)a[1] << 16); + uint a1 = (uint)a[2] | ((uint)a[3] << 16); + uint tmp = (uint)a[4] | ((uint)a[5] << 16); + + uint g = is >> 2; + uint w = (g & 1u) ? a1 : a0; + w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4); + + uint sc6 = (w >> (8*(is & 3))) & 0xFFu; + float dl = (float)src0_d[ib] * ((float)sc6 - 32.0f); + + uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16); + uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16); + + float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3, + (q.s1 >> shift) & 3, + (q.s2 >> shift) & 3, + (q.s3 >> shift) & 3)); + float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f, + (hb.s1 & m) ? 0.f : 4.f, + (hb.s2 & m) ? 0.f : 4.f, + (hb.s3 & m) ? 0.f : 4.f); + float4 v1 = (qv - hv) * dl; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int a = 0; a < TM/4; a++) { + cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a); + } + + for (int cc = 0; cc < TN; cc++) { + const float cache_b = buf_b[(i) * BN + th_c * TN + cc]; + for (int a = 0; a < TM/4; a++) { + const int sums_idx = cc*(TM/4) + a; + sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int a = 0; a < TM/4; a++) { + const float4 v = sums4[cc * (TM/4) + a]; + const float vs[4] = { v.s0, v.s1, v.s2, v.s3 }; + for (int k = 0; k < 4; k++) { + if (dr + 4*a + k < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k]; + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl new file mode 100644 index 0000000000..77c70f4ac4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl @@ -0,0 +1,197 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q3_K * x = (global block_q3_K *) (src0 + offset_src0); + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(x[i].qs + q_offset); + global ushort * h = (global ushort *)(x[i].hmask + l0); + global ushort * a = (global ushort *)(x[i].scales); + global half * dh = &x[i].d; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += nb01/2; + h += nb01/2; + a += nb01/2; + dh += nb01/2; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl new file mode 100644 index 0000000000..c2c2219d8a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl @@ -0,0 +1,199 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32_flat( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K); + ulong row_stride = nb01/sizeof(block_q3_K); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset); + global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0); + global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i)); + global half * dh = src0_d + offset_src0 + i; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += row_stride*32; + h += row_stride*16; + a += row_stride*6; + dh += row_stride*1; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index d4c257cf78..77a17412ff 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -196,7 +196,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { } } -static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +template +static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -204,25 +205,25 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * const int32_t dim = ((int32_t *) dst->op_params)[0]; - GGML_ASSERT(src0->type == GGML_TYPE_Q4_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q4_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q4_0); - GGML_ASSERT(src0->ne[0] % QK4_0 == 0); - GGML_ASSERT(src1->ne[0] % QK4_0 == 0); - GGML_ASSERT(dst->ne[0] % QK4_0 == 0); + GGML_ASSERT(src0->type == qtype); + GGML_ASSERT(src1->type == qtype); + GGML_ASSERT(dst->type == qtype); + GGML_ASSERT(src0->ne[0] % qk == 0); + GGML_ASSERT(src1->ne[0] % qk == 0); + GGML_ASSERT(dst->ne[0] % qk == 0); - const int ne00_blk = src0->ne[0] / QK4_0; - const int ne0_blk = dst->ne[0] / QK4_0; + const int ne00_blk = src0->ne[0] / qk; + const int ne0_blk = dst->ne[0] / qk; if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q4_0 * src0_d = (const block_q4_0 *) src0->data; - const block_q4_0 * src1_d = (const block_q4_0 *) src1->data; - block_q4_0 * dst_d = (block_q4_0 *) dst->data; - const size_t type_size = sizeof(block_q4_0); + const block_t * src0_d = (const block_t *) src0->data; + const block_t * src1_d = (const block_t *) src1->data; + block_t * dst_d = (block_t *) dst->data; + const size_t type_size = sizeof(block_t); if (dim != 3) { for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl( + concat_T_sycl( src0_d + i3 * (src0->nb[3] / type_size), src1_d + i3 * (src1->nb[3] / type_size), dst_d + i3 * (dst->nb[3] / type_size), @@ -237,223 +238,12 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); } } else { - concat_T_sycl_non_cont( + concat_T_sycl_non_cont( stream, (const char *) src0->data, (const char *) src1->data, (char *) dst->data, ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q4_1); - GGML_ASSERT(src1->type == GGML_TYPE_Q4_1); - GGML_ASSERT(dst->type == GGML_TYPE_Q4_1); - GGML_ASSERT(src0->ne[0] % QK4_1 == 0); - GGML_ASSERT(src1->ne[0] % QK4_1 == 0); - GGML_ASSERT(dst->ne[0] % QK4_1 == 0); - - const int ne00_blk = src0->ne[0] / QK4_1; - const int ne0_blk = dst->ne[0] / QK4_1; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q4_1 * src0_d = (const block_q4_1 *) src0->data; - const block_q4_1 * src1_d = (const block_q4_1 *) src1->data; - block_q4_1 * dst_d = (block_q4_1 *) dst->data; - const size_t type_size = sizeof(block_q4_1); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q5_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q5_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q5_0); - GGML_ASSERT(src0->ne[0] % QK5_0 == 0); - GGML_ASSERT(src1->ne[0] % QK5_0 == 0); - GGML_ASSERT(dst->ne[0] % QK5_0 == 0); - - const int ne00_blk = src0->ne[0] / QK5_0; - const int ne0_blk = dst->ne[0] / QK5_0; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q5_0 * src0_d = (const block_q5_0 *) src0->data; - const block_q5_0 * src1_d = (const block_q5_0 *) src1->data; - block_q5_0 * dst_d = (block_q5_0 *) dst->data; - const size_t type_size = sizeof(block_q5_0); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q5_1); - GGML_ASSERT(src1->type == GGML_TYPE_Q5_1); - GGML_ASSERT(dst->type == GGML_TYPE_Q5_1); - GGML_ASSERT(src0->ne[0] % QK5_1 == 0); - GGML_ASSERT(src1->ne[0] % QK5_1 == 0); - GGML_ASSERT(dst->ne[0] % QK5_1 == 0); - - const int ne00_blk = src0->ne[0] / QK5_1; - const int ne0_blk = dst->ne[0] / QK5_1; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q5_1 * src0_d = (const block_q5_1 *) src0->data; - const block_q5_1 * src1_d = (const block_q5_1 *) src1->data; - block_q5_1 * dst_d = (block_q5_1 *) dst->data; - const size_t type_size = sizeof(block_q5_1); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3], - src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], - ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], - dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); - } -} - -static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - queue_ptr stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src1->type == GGML_TYPE_Q8_0); - GGML_ASSERT(dst->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src0->ne[0] % QK8_0 == 0); - GGML_ASSERT(src1->ne[0] % QK8_0 == 0); - GGML_ASSERT(dst->ne[0] % QK8_0 == 0); - - const int ne00_blk = src0->ne[0] / QK8_0; - const int ne0_blk = dst->ne[0] / QK8_0; - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const block_q8_0 * src0_d = (const block_q8_0 *) src0->data; - const block_q8_0 * src1_d = (const block_q8_0 *) src1->data; - block_q8_0 * dst_d = (block_q8_0 *) dst->data; - const size_t type_size = sizeof(block_q8_0); - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_T_sycl( - src0_d + i3 * (src0->nb[3] / type_size), - src1_d + i3 * (src1->nb[3] / type_size), - dst_d + i3 * (dst->nb[3] / type_size), - ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, - dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); - } - } else { - concat_T_sycl_non_cont( - stream, (const char *) src0->data, (const char *) src1->data, - (char *) dst->data, - ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], - src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], - src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3], + src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3], src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); @@ -487,19 +277,19 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { concat_impl_sycl(ctx, dst); break; case GGML_TYPE_Q4_0: - concat_impl_q4_0_sycl(ctx, dst); + concat_impl_q_sycl(ctx, dst); break; case GGML_TYPE_Q4_1: - concat_impl_q4_1_sycl(ctx, dst); + concat_impl_q_sycl(ctx, dst); break; case GGML_TYPE_Q5_0: - concat_impl_q5_0_sycl(ctx, dst); + concat_impl_q_sycl(ctx, dst); break; case GGML_TYPE_Q5_1: - concat_impl_q5_1_sycl(ctx, dst); + concat_impl_q_sycl(ctx, dst); break; case GGML_TYPE_Q8_0: - concat_impl_q8_0_sycl(ctx, dst); + concat_impl_q_sycl(ctx, dst); break; default: fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type)); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index f3782c2cb3..97ad492e59 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -842,6 +842,19 @@ catch (sycl::exception const &exc) { std::exit(1); } +// allocates the pinned upload staging on first use; returns false if staging is disabled or the allocation failed +static bool ggml_sycl_buffer_init_staging(ggml_backend_sycl_buffer_context * ctx) { + const int n_slots = g_ggml_sycl_upload_staging_slots; + if (n_slots > 0 && ctx->staging.data == nullptr) { + ctx->staging.data = sycl::malloc_host(n_slots * ctx->staging_slot_size, *ctx->stream); + if (ctx->staging.data != nullptr) { + ctx->staging.events.resize(n_slots); + ctx->staging.submitted.assign(n_slots, false); + } + } + return ctx->staging.data != nullptr; +} + static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, @@ -854,15 +867,7 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, // copy through pinned memory so the device never reads mmap()ed pages directly // chunks pipeline on the in-order compute queue, so no drain per tensor is needed - const int n_slots = g_ggml_sycl_upload_staging_slots; - if (n_slots > 0 && ctx->staging.data == nullptr) { - ctx->staging.data = sycl::malloc_host(n_slots * ctx->staging_slot_size, *ctx->stream); - if (ctx->staging.data != nullptr) { - ctx->staging.events.resize(n_slots); - ctx->staging.submitted.assign(n_slots, false); - } - } - if (ctx->staging.data != nullptr) { + if (ggml_sycl_buffer_init_staging(ctx)) { queue_ptr stream = ctx->stream; char * dst = (char *) tensor->data + offset; const char * src = (const char *) data; @@ -904,6 +909,49 @@ catch (sycl::exception const &exc) { std::exit(1); } +// Copy n_copies rows in one call. -sm tensor uploads split weights this way; the generic fallback does one set_tensor per row. +static void ggml_backend_sycl_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, + const void * data, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, + size_t stride_data) try { + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context; + const char * src = (const char *) data; + ggml_sycl_set_device(ctx->device); + + if (stride_tensor != size || size > ctx->staging_slot_size || !ggml_sycl_buffer_init_staging(ctx)) { + // dst rows not contiguous, row larger than a staging slot, or no staging: same as the generic fallback + for (size_t i = 0; i < n_copies; ++i) { + ggml_backend_sycl_buffer_set_tensor(buffer, tensor, src + i * stride_data, offset + i * stride_tensor, size); + } + return; + } + + // dst rows are contiguous: pack as many rows as fit into a staging slot and copy each slot with one memcpy + // a slot is only waited on when it is reused, so packing the next slot overlaps with the previous copies + queue_ptr stream = ctx->stream; + char * dst = (char *) tensor->data + offset; + const size_t rows_per_slot = ctx->staging_slot_size / size; + for (size_t i0 = 0; i0 < n_copies; i0 += rows_per_slot) { + const size_t n = std::min(rows_per_slot, n_copies - i0); + const int slot = ctx->staging.next; + ctx->staging.next = (ctx->staging.next + 1) % (int) ctx->staging.submitted.size(); + if (ctx->staging.submitted[slot]) { + ctx->staging.events[slot].wait_and_throw(); + } + char * stage = (char *) ctx->staging.data + slot * ctx->staging_slot_size; + for (size_t i = 0; i < n; ++i) { + memcpy(stage + i * size, src + (i0 + i) * stride_data, size); + } + ctx->staging.events[slot] = stream->memcpy(dst + i0 * size, stage, n * size); + ctx->staging.submitted[slot] = true; + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + static void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor *tensor, void *data, size_t offset, @@ -1122,7 +1170,7 @@ static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .memset_tensor = */ ggml_backend_sycl_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_sycl_buffer_set_tensor, /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, + /* .set_tensor_2d = */ ggml_backend_sycl_buffer_set_tensor_2d, /* .get_tensor_2d = */ NULL, /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, @@ -4414,7 +4462,15 @@ static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { - void * ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + // async_malloc throws on out-of-memory; return nullptr like the sync path so callers can fall back. + void * ptr = nullptr; + try { + ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + } catch (const sycl::exception & e) { + GGML_LOG_WARN("%s: async_malloc of %zu bytes failed: %s\n", __func__, size, e.what()); + ggml_sycl_memtrace_fail(GGML_SYCL_MEM_ASYNC, size); + return nullptr; + } ggml_sycl_memtrace_add(GGML_SYCL_MEM_ASYNC, ptr, size); return ptr; } @@ -4481,307 +4537,317 @@ private: bool host_fallback = false; }; -static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, - dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); +// Reorder a 2D tensor of quantized blocks in place into [head of every block][tail of every block] without a +// full-size temporary copy. The head is the contiguous `head_bytes` at `head_offset` inside each block. Heads only +// move towards the front, so they are packed in chunks that never overwrite unread blocks. The tail overlaps +// blocks that are not packed yet, so `write_tail(x, ib, tail, nblocks)` first writes every block's remaining fields +// (in their final layout, relative to the start of the tail) into a side buffer that is copied to the end afterwards. +template +static bool reorder_qw_planar(const char * name, uint8_t * data_device, size_t size, size_t head_offset, + size_t head_bytes, tail_fn_t write_tail, dpct::queue_ptr stream) { + GGML_ASSERT(size % sizeof(block_t) == 0); + GGML_ASSERT(head_offset + head_bytes <= sizeof(block_t)); + + const int64_t nblocks = size / sizeof(block_t); + const size_t tail_bytes = (sizeof(block_t) - head_bytes) * nblocks; + + sycl_reorder_temp_buffer tail_tmp(stream, tail_bytes); + if (!tail_tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, tail_bytes); + return false; + } + uint8_t * tail_buf = static_cast(tail_tmp.ptr); + + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_blocks = std::max(1, std::min(nblocks, max_chunk_bytes / sizeof(block_t))); + sycl_reorder_temp_buffer tmp(stream, chunk_blocks * sizeof(block_t)); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, + (size_t) chunk_blocks * sizeof(block_t)); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + sycl::event event = stream->parallel_for(nblocks, [=](auto i) { + const int64_t ib = i; + write_tail(((const block_t *) data_device)[ib], ib, tail_buf, nblocks); + }); if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); + event.wait_and_throw(); } - GGML_ASSERT((size % sizeof(block_q4_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q4_0) == 0)); - int offset_blks = offset / sizeof(block_q4_0); - auto qs_ptr = data_device + offset_blks * QK4_0 / 2; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows / 2) + offset_blks; + for (int64_t b0 = 0; b0 < nblocks; b0 += chunk_blocks) { + const int64_t n_chunk = std::min(chunk_blocks, nblocks - b0); - auto reorder_event = stream->parallel_for( - size / sizeof(block_q4_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q4_0* x = (const block_q4_0*)tmp_buf; - const int ib = i; + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(tmp_buf, data_device + b0 * sizeof(block_t), + n_chunk * sizeof(block_t)))); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait(); + } - for (int j = 0; j < QK4_0/2; j ++) - { - *(qs_ptr + ib * QK4_0 / 2 + j) = x[ib].qs[j]; + uint8_t * head_ptr = data_device + b0 * head_bytes; + event = stream->parallel_for(n_chunk, [=](auto i) { + const int64_t ib = i; + const uint8_t * src = tmp_buf + ib * sizeof(block_t) + head_offset; + for (size_t j = 0; j < head_bytes; ++j) { + head_ptr[ib * head_bytes + j] = src[j]; } - *(d_ptr + ib) = x[ib].d; }); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait_and_throw(); + } + } + + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(data_device + head_bytes * nblocks, tail_buf, tail_bytes))); if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + event.wait(); } return true; } +static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, + dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows / 2 == size / sizeof(block_q4_0) * (QK4_0 / 2)); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q4_0, qs), QK4_0 / 2, + [](const block_q4_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); +} + static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - GGML_ASSERT((size % sizeof(block_q8_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q8_0) == 0)); - int offset_blks = offset / sizeof(block_q8_0); - auto qs_ptr = data_device + offset_blks * QK8_0; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows) + offset_blks; - - auto reorder_event = stream->parallel_for( - size / sizeof(block_q8_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q8_0* x = (const block_q8_0*)tmp_buf; - const int ib = i; - - for (int j = 0; j < QK8_0; j++) - { - *((int8_t*)qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; - } - *(d_ptr + ib) = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; -} - -// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout. -static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { - GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; - - sycl_reorder_temp_buffer tmp(stream, total_bytes); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; - - auto * qs_ptr = base; - auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0); - - for (int j = 0; j < QK8_0; ++j) { - *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; - } - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows == size / sizeof(block_q8_0) * QK8_0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q8_0, qs), QK8_0, + [](const block_q8_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); } static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q4_K) == 0); - GGML_ASSERT(offset % sizeof(block_q4_K) == 0); + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q4_K, qs), QK_K / 2, + [](const block_q4_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); +} - const int nblocks = size / sizeof(block_q4_K); +// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout, a few experts at a time. +static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, size); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - auto * qs_ptr = data_device; - auto * scales_ptr = qs_ptr + QK_K / 2 * nblocks; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q4_K * x = (const block_q4_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); } - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; - } + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + auto * qs_ptr = base; + auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0); + + for (int j = 0; j < QK8_0; ++j) { + *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } } return true; } // Reorder each expert slice into a self-contained SoA layout. +// Like the MXFP4 version below, this goes a few experts at a time through a small temporary: +// a whole-tensor temporary (GBs for large MoE models) can exceed the VRAM left on a nearly full card. static bool reorder_qw_q4_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q4_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q4_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q4_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q4_K * x = (const block_q4_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; - - auto * qs_ptr = base; - auto * scales_ptr = qs_ptr + QK_K / 2 * blocks_per_expert; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); - - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); } - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q4_K * x = (const block_q4_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; + + auto * qs_ptr = base; + auto * scales_ptr = qs_ptr + QK_K / 2 * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -// Reorder each Q5_K expert slice into [qs][qh][scales][dm]. +// Reorder each Q5_K expert slice into [qs][qh][scales][dm], a few experts at a time (see Q4_K above). static bool reorder_qw_q5_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q5_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q5_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q5_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q5_K * x = (const block_q5_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto * qs_ptr = base; - auto * qh_ptr = qs_ptr + (QK_K / 2) * blocks_per_expert; - auto * scales_ptr = qh_ptr + (QK_K / 8) * blocks_per_expert; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q5_K * x = (const block_q5_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + auto * qs_ptr = base; + auto * qh_ptr = qs_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 8) * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - for (int j = 0; j < QK_K / 8; ++j) { - qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; - } - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; - } - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -// Reorder each Q6_K expert slice into [ql][qh][scales][d]. +// Reorder each Q6_K expert slice into [ql][qh][scales][d], a few experts at a time (see Q4_K above). static bool reorder_qw_q6_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { GGML_ASSERT(expert_bytes % sizeof(block_q6_K) == 0); - const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q6_K)); - const size_t total_bytes = expert_bytes * (size_t) n_expert; + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q6_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, total_bytes); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - const int total_blocks = blocks_per_expert * (int) n_expert; - auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { - const int gb = gb_; - const int e = gb / blocks_per_expert; - const int ib = gb % blocks_per_expert; - const block_q6_K * x = (const block_q6_K *) (tmp_buf + (size_t) e * expert_bytes); - uint8_t * base = data_device + (size_t) e * expert_bytes; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto * ql_ptr = base; - auto * qh_ptr = ql_ptr + (QK_K / 2) * blocks_per_expert; - auto * scales_ptr = qh_ptr + (QK_K / 4) * blocks_per_expert; - auto * d_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * blocks_per_expert); + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q6_K * x = (const block_q6_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - ql_ptr[ib * (QK_K / 2) + j] = x[ib].ql[j]; + auto * ql_ptr = base; + auto * qh_ptr = ql_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 4) * blocks_per_expert; + auto * d_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + ql_ptr[ib * (QK_K / 2) + j] = x[ib].ql[j]; + } + for (int j = 0; j < QK_K / 4; ++j) { + qh_ptr[ib * (QK_K / 4) + j] = x[ib].qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - for (int j = 0; j < QK_K / 4; ++j) { - qh_ptr[ib * (QK_K / 4) + j] = x[ib].qh[j]; - } - for (int j = 0; j < QK_K / 16; ++j) { - scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; - } - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } @@ -4839,278 +4905,89 @@ static bool reorder_qw_mxfp4_moe(uint8_t * data_device, size_t expert_bytes, int } static bool reorder_qw_q2_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q2_K) == 0); - GGML_ASSERT(offset % sizeof(block_q2_K) == 0); - - const int nblocks = size / sizeof(block_q2_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * scales_ptr = qs_ptr + (QK_K / 4) * nblocks; - sycl::half2 * dm_ptr = (sycl::half2 *) (scales_ptr + (QK_K / 16) * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q2_K * x = (const block_q2_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 4; ++j) { - qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j]; - } - - for (int j = 0; j < QK_K / 16; ++j) { - scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; - } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q2_K, qs), QK_K / 4, + [](const block_q2_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + (QK_K / 16) * nblocks))[ib] = x.dm; + }, stream); } static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q3_K) == 0); - GGML_ASSERT(offset % sizeof(block_q3_K) == 0); - - const int nblocks = size / sizeof(block_q3_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * hmask_ptr = qs_ptr + (QK_K / 4) * nblocks; - auto * scales_ptr = hmask_ptr + (QK_K / 8) * nblocks; - sycl::half * d_ptr = (sycl::half *) (scales_ptr + 12 * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q3_K * x = (const block_q3_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 4; ++j) { - qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j]; - } - - for (int j = 0; j < QK_K / 8; ++j) { - hmask_ptr[ib * (QK_K / 8) + j] = x[ib].hmask[j]; - } - - for (int j = 0; j < 12; ++j) { - scales_ptr[ib * 12 + j] = x[ib].scales[j]; - } - - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q3_K, qs), QK_K / 4, + [](const block_q3_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * hmask = tail; + uint8_t * scales = hmask + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + hmask[ib * (QK_K / 8) + j] = x.hmask[j]; + } + for (int j = 0; j < 12; ++j) { + scales[ib * 12 + j] = x.scales[j]; + } + ((sycl::half *) (scales + 12 * nblocks))[ib] = x.d; + }, stream); } static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q5_K) == 0); - GGML_ASSERT(offset % sizeof(block_q5_K) == 0); - - const int nblocks = size / sizeof(block_q5_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * qh_ptr = qs_ptr + (QK_K / 2) * nblocks; - auto * scales_ptr = qh_ptr + (QK_K / 8) * nblocks; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q5_K * x = (const block_q5_K *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } - - for (int j = 0; j < QK_K / 8; ++j) { - qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; - } - - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; - } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q5_K, qs), QK_K / 2, + [](const block_q5_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + qh[ib * (QK_K / 8) + j] = x.qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); } static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q6_K) == 0); - GGML_ASSERT(offset % sizeof(block_q6_K) == 0); - - const int nblocks = size / sizeof(block_q6_K); - - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * ql_ptr = data_device; - auto * qh_ptr = ql_ptr + (QK_K / 2) * nblocks; - auto * scales_ptr = qh_ptr + (QK_K / 4) * nblocks; - sycl::half * dm_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q6_K * x = (const block_q6_K *) tmp_buf; - const int ib = i; - - const uint8_t * ql = x[ib].ql; - const uint8_t * qh = x[ib].qh; - uint8_t * base_ql_ptr = ql_ptr + (QK_K / 2) * ib; - uint8_t * base_qh_ptr = qh_ptr + (QK_K / 4) * ib; - uint8_t * base_scales_ptr = scales_ptr + (QK_K / 16) * ib; - - for (int j = 0; j < QK_K / 2; ++j) { - base_ql_ptr[j] = ql[j]; - } - for (int j = 0; j < QK_K / 4; ++j) { - base_qh_ptr[j] = qh[j]; - } - - for (int j = 0; j < QK_K / 16; ++j) { - base_scales_ptr[j] = x[ib].scales[j]; - } - - dm_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q6_K, ql), QK_K / 2, + [](const block_q6_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 4) * nblocks; + for (int j = 0; j < QK_K / 4; ++j) { + qh[ib * (QK_K / 4) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half *) (scales + (QK_K / 16) * nblocks))[ib] = x.d; + }, stream); } static bool reorder_qw_iq3_s(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_iq3_s) == 0); - - const int nblocks = size / sizeof(block_iq3_s); - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * qh_ptr = qs_ptr + (QK_K / 4) * nblocks; - auto * signs_ptr = qh_ptr + (QK_K / 32) * nblocks; - auto * metadata_ptr = signs_ptr + (QK_K / 8) * nblocks; - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_iq3_s * x = (const block_iq3_s *) tmp_buf; - const int ib = i; - - for (int j = 0; j < QK_K / 4; ++j) { - qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j]; - } - for (int j = 0; j < QK_K / 32; ++j) { - qh_ptr[ib * (QK_K / 32) + j] = x[ib].qh[j]; - } - for (int j = 0; j < QK_K / 8; ++j) { - signs_ptr[ib * (QK_K / 8) + j] = x[ib].signs[j]; - } - - uint8_t * metadata = metadata_ptr + ib * (sizeof(ggml_half) + IQ3S_N_SCALE); - *reinterpret_cast(metadata) = x[ib].d; - for (int j = 0; j < IQ3S_N_SCALE; ++j) { - metadata[sizeof(ggml_half) + j] = x[ib].scales[j]; - } - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + return reorder_qw_planar(__func__, data_device, size, offsetof(block_iq3_s, qs), QK_K / 4, + [](const block_iq3_s & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * signs = qh + (QK_K / 32) * nblocks; + uint8_t * metadata = signs + (QK_K / 8) * nblocks + ib * (sizeof(ggml_half) + IQ3S_N_SCALE); + for (int j = 0; j < QK_K / 32; ++j) { + qh[ib * (QK_K / 32) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + signs[ib * (QK_K / 8) + j] = x.signs[j]; + } + *reinterpret_cast(metadata) = x.d; + for (int j = 0; j < IQ3S_N_SCALE; ++j) { + metadata[sizeof(ggml_half) + j] = x.scales[j]; + } + }, stream); } static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_iq3_xxs) == 0); - - const int nblocks = size / sizeof(block_iq3_xxs); - sycl_reorder_temp_buffer tmp(stream, size); - if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); - return false; - } - uint8_t * tmp_buf = static_cast(tmp.ptr); - - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } - - auto * qs_ptr = data_device; - auto * d_ptr = reinterpret_cast(qs_ptr + (3 * QK_K / 8) * nblocks); - - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_iq3_xxs * x = (const block_iq3_xxs *) tmp_buf; - const int ib = i; - - for (int j = 0; j < 3 * QK_K / 8; ++j) { - qs_ptr[ib * (3 * QK_K / 8) + j] = x[ib].qs[j]; - } - d_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); - } - return true; + return reorder_qw_planar(__func__, data_device, size, offsetof(block_iq3_xxs, qs), 3 * QK_K / 8, + [](const block_iq3_xxs & x, int64_t ib, uint8_t * tail, int64_t) { + reinterpret_cast(tail)[ib] = x.d; + }, stream); } static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { @@ -7626,8 +7503,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: - // the scale kernel is f32 only - return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; + // the scale kernel is f32 only, and reads src0 flat + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_CONT: return true; case GGML_OP_TRI: diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 97ce165c62..c3853814aa 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -369,6 +369,45 @@ static void mul_mat_vec_q_ncols( } } +template +static void launch_mul_mat_vec_q(const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % assert_qk == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + const auto launch = [&](auto ncols_tag) { + constexpr int N = decltype(ncols_tag)::value; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + if constexpr (N == 1) { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + } else { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + } + }); + }); + }; + + switch (ncols_dst) { + case 1: launch(std::integral_constant{}); break; + case 2: launch(std::integral_constant{}); break; + case 3: launch(std::integral_constant{}); break; + case 4: launch(std::integral_constant{}); break; + case 5: launch(std::integral_constant{}); break; + case 6: launch(std::integral_constant{}); break; + case 7: launch(std::integral_constant{}); break; + case 8: launch(std::integral_constant{}); break; + default: GGML_ABORT("unsupported ncols_dst=%d", ncols_dst); + } +} + template static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx, const void *__restrict__ vy, @@ -868,364 +907,136 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q4_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q4_1_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_MXFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_mxfp4_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_MXFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_NVFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_nvfp4_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_NVFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_nvfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_nvfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_nvfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_nvfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_nvfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_nvfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_nvfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_nvfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q5_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q5_1_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK8_0 == 0); @@ -1357,248 +1168,98 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q8_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q1_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q2_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q2_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q2_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q2_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q2_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q2_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q2_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q2_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q2_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q2_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1662,26 +1323,12 @@ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } + static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1741,95 +1388,30 @@ static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols( } } -template -static void mul_mat_vec_q3_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} - static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q3_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q3_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q3_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q3_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q3_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q3_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q3_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q3_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q4_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -1837,19 +1419,15 @@ static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q4_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q4_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q4_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q4_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q4_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q4_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q4_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q4_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -1931,51 +1509,11 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q5_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -1983,19 +1521,15 @@ static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q5_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q5_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q5_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q5_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q5_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q5_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q5_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q5_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); @@ -2133,51 +1667,11 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); } -template -static void mul_mat_vec_q6_K_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, - stride_col_y, stride_col_dst, item_ct1); - }); - }); -} static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, @@ -2185,20 +1679,16 @@ static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols( const int ncols_dst, const int stride_col_y, const int stride_col_dst, dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_q6_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_q6_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_q6_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_q6_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_q6_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_q6_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_q6_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_q6_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst); + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index 5cf46c8908..a1505555aa 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -848,18 +848,18 @@ struct vk_device_struct { vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; - vk_pipeline pipeline_rms_norm_f32; - vk_pipeline pipeline_rms_norm_mul_f32; - vk_pipeline pipeline_rms_norm_mul_add_f32; - vk_pipeline pipeline_rms_norm_mul_add_mul_f32; + std::array pipeline_rms_norm_f32; + std::array pipeline_rms_norm_mul_f32; + std::array pipeline_rms_norm_mul_add_f32; + std::array pipeline_rms_norm_mul_add_mul_f32; vk_pipeline pipeline_rms_norm_mul_add_partials_f32; vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f16; vk_pipeline pipeline_rms_norm_partials_f32; vk_pipeline pipeline_rms_norm_mul_partials_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f32; - vk_pipeline pipeline_rms_norm_mul_rope_f32_f16; + std::array pipeline_rms_norm_mul_rope_f32_f32; + std::array pipeline_rms_norm_mul_rope_f32_f16; vk_pipeline pipeline_rms_norm_back_f32; vk_pipeline pipeline_l2_norm_f32; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 33936acae8..14b2cf3849 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3256,21 +3256,34 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_f32, "rms_norm_mul_add_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_f32, "rms_norm_mul_add_mul_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); + const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support; + const uint32_t rms_set_rows_block_size = 128; +#define RMS_SHADER(name) (rms_subgroups ? name##_subgroup_len : name##_len), (rms_subgroups ? name##_subgroup_data : name##_data) + // 128/256/512 block sizes, picked by row length + for (uint32_t i = 0; i < 3; ++i) { + const uint32_t block_size = 128u << i; + const std::string suffix = "_" + std::to_string(block_size); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size}, 1, true, rms_subgroups); + } ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", rms_norm_set_rows_f32_f32_len, rms_norm_set_rows_f32_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", rms_norm_set_rows_f32_f16_len, rms_norm_set_rows_f32_f16_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", RMS_SHADER(rms_norm_set_rows_f32_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", RMS_SHADER(rms_norm_set_rows_f32_f16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true); if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) { - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true); + for (uint32_t i = 0; i < 3; ++i) { + const uint32_t block_size = 128u << i; + const std::string suffix = "_" + std::to_string(block_size); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f32[i], "rms_norm_mul_rope_f32_f32" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f32), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f16[i], "rms_norm_mul_rope_f32_f16" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f16), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + } } +#undef RMS_SHADER ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); @@ -8740,6 +8753,16 @@ static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, return ggml_vk_conv_select_shape(ctx, K, NPQ); } +static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array & pipelines, int64_t ncols) { + if (pipelines[0] && ncols <= 128) { + return pipelines[0]; + } + if (pipelines[1] && ncols <= 256) { + return pipelines[1]; + } + return pipelines[2]; +} + static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_GET_ROWS: @@ -8980,7 +9003,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (ctx->do_add_rms_partials) { return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; } - return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32; + return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]); } return nullptr; case GGML_OP_RMS_NORM_BACK: @@ -10942,8 +10965,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s pipeline = do_post_multiply ? ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32; } else { - pipeline = do_post_multiply ? - ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32; + pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ? + ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]); } ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); if (ctx->do_add_rms_partials) { @@ -11016,7 +11039,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s pc.bin = bin; pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride); - vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; + const auto & pipelines = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32; + vk_pipeline pipeline = ggml_vk_rms_norm_pipeline(pipelines, src0->ne[0]); ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp index 314c8e565e..6a2525849c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp @@ -3,6 +3,15 @@ #include "generic_binary_head.glsl" #include "types.glsl" +#extension GL_EXT_control_flow_attributes : enable +#if USE_SUBGROUP_ADD +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#endif + +layout (constant_id = 3) const int BLOCK_SIZE = 512; +layout(local_size_x_id = 3, local_size_y = 1, local_size_z = 1) in; + #if RMS_NORM_ROPE_FUSION layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; @@ -11,7 +20,8 @@ layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; // data is passed from rms_norm -> rope through shared memory. // rms_norm calls this data_d, rope calls this rope_data_a. // Binding 2 is not used -shared FLOAT_TYPE rope_data_a[1024]; +// host picks a block size such that ncols <= 2*BLOCK_SIZE +shared FLOAT_TYPE rope_data_a[2*BLOCK_SIZE]; #define data_d rope_data_a layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];}; @@ -38,16 +48,11 @@ layout (binding = 3) readonly buffer I {uvec2 data_i[];}; #endif -#extension GL_EXT_control_flow_attributes : enable -#define BLOCK_SIZE 512 - layout (constant_id = 1) const bool do_multiply = false; #if RMS_NORM_ADD_FUSION layout (constant_id = 2) const bool do_post_multiply = false; #endif -layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; - shared FLOAT_TYPE sumsh[BLOCK_SIZE]; void rms_norm(uint num_iters) { @@ -88,17 +93,38 @@ void rms_norm(uint num_iters) { sum += xi * xi; } - sumsh[tid] = sum; - // sum up partial sums and write back result - barrier(); - [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) { - sum += sumsh[tid + s]; - sumsh[tid] = sum; + #if USE_SUBGROUP_ADD + sum = subgroupAdd(sum); + if (subgroupElect()) { + sumsh[gl_SubgroupID] = sum; } barrier(); - } - sum = sumsh[0]; + sum = FLOAT_TYPE(0.0f); + if(gl_NumSubgroups <= 4) { // fast path to skip the 2nd subgroupAdd + [[unroll]] for(uint i=0; i < gl_NumSubgroups; ++i) { + sum += sumsh[i]; + } + } + else { + // Each subgroup reads the partials, so one workgroup barrier is enough. + for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) { + sum += sumsh[i]; + } + sum = subgroupAdd(sum); + } + #else + sumsh[tid] = sum; + // sum up partial sums and write back result + barrier(); + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum += sumsh[tid + s]; + sumsh[tid] = sum; + } + barrier(); + } + sum = sumsh[0]; + #endif const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols); const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index f545015464..3af09fb48a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -853,14 +853,18 @@ void process_shaders() { // Norms string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_mul_add_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); + for (bool use_subgroups : {false, true}) { + const std::string suffix = use_subgroups ? "_subgroup" : ""; + const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); + string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict); + string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}})); + string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE' + string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); + string_to_spv("rms_norm_mul_rope_f32_f16" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}})); + } string_to_spv("rms_norm_mul_add_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}})); - string_to_spv("rms_norm_set_rows_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); - string_to_spv("rms_norm_set_rows_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}})); string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); - string_to_spv("rms_norm_mul_rope_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); - string_to_spv("rms_norm_mul_rope_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}})); string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index ed8554b141..5d8449a883 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -3503,7 +3503,7 @@ static struct ggml_tensor * ggml_scale_impl( float s, float b, bool inplace) { - GGML_ASSERT(ggml_is_padded_1d(a)); + GGML_ASSERT(ggml_is_contiguous_rows(a)); struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a); diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 06fc56bb27..fe3ad3eb77 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1347,6 +1347,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MISTRAL4: case LLM_ARCH_MELLUM: // [TAG_MOE_AMAX_SCALE] case LLM_ARCH_FLINT: // [TAG_MOE_AMAX_SCALE] + case LLM_ARCH_LAGUNA: // [TAG_MOE_AMAX_SCALE] case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_K3: diff --git a/src/llama-context.cpp b/src/llama-context.cpp index d5a66a4999..455f5198e4 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -489,6 +489,7 @@ llama_context::llama_context( /*.swa_full =*/ params.swa_full, /*.ctx_type =*/ cparams.ctx_type, /*.mem_other =*/ llama_get_memory(cparams.ctx_other), + /*.no_alloc =*/ model.no_alloc(), }; memory.reset(model.create_memory(params_mem, cparams)); @@ -559,7 +560,7 @@ llama_context::llama_context( } if (cparams.moe_cache_size > 0) { - moe_cache = std::make_unique(model, backend_ptrs, backend_buft, cparams.moe_cache_size); + moe_cache = std::make_unique(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size); } sched_reserve(); @@ -587,7 +588,7 @@ llama_context::~llama_context() { synchronize(); // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation - if (!model.hparams.no_alloc && !opt_ctx) { + if (!model.no_alloc() && !opt_ctx) { for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; @@ -788,7 +789,7 @@ void llama_context::sched_reserve() { // reserve pp (prompt processing) graph first so that buffers are only allocated once { auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), - model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr); + model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr); if (!gf) { if (cparams.pipeline_parallel) { LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__); @@ -815,7 +816,7 @@ void llama_context::sched_reserve() { n_splits_tg = n_splits_pp; n_nodes_tg = n_nodes_pp; } else { - auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc); + auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc()); if (!gf) { throw std::runtime_error("failed to allocate compute tg buffers"); } @@ -837,10 +838,10 @@ void llama_context::sched_reserve() { // [TAG_RESERVE_DIAG_DECAY] // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1` - gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc); + gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.no_alloc()); break; default: - gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc); + gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc()); }; if (!gf) { @@ -851,7 +852,7 @@ void llama_context::sched_reserve() { for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; ggml_backend_buffer_type_t buft = backend_buft[i]; - if (!model.hparams.no_alloc) { + if (!model.no_alloc()) { backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend); } if (backend_buf_exp_size[i] > 1) { @@ -3730,7 +3731,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const { ret[buft].context += size; } } - if (model.hparams.no_alloc) { + if (model.no_alloc()) { for (size_t i = 0; i < backends.size(); ++i) { ggml_backend_t backend = backends[i].get(); ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index a22dfba965..7309705dc4 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2519,7 +2519,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( // ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922 llm_graph_amax_src1 amax_src1 = { nullptr, nullptr }; - if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) { + if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_LAGUNA) { amax_src1 = build_amax_src1_f16(cur); cur = amax_src1.res; } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 760bb8ac67..7abb372cea 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -61,7 +61,6 @@ struct llama_hparams { // for example: n_embd_out() vs n_embd_out_impl bool vocab_only; - bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams bool rope_finetuned; bool use_par_res; bool swin_norm; diff --git a/src/llama-kv-cache-dsa-iswa.cpp b/src/llama-kv-cache-dsa-iswa.cpp index 784c78aa78..3d91228f3b 100644 --- a/src/llama-kv-cache-dsa-iswa.cpp +++ b/src/llama-kv-cache-dsa-iswa.cpp @@ -13,6 +13,7 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -22,7 +23,6 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa( uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, const layer_filter_cb & filter_mla, const layer_filter_cb & filter_lid, const layer_reuse_cb & reuse) : unified(unified) { @@ -63,15 +63,15 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa( LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa); kv_dsa = std::make_unique( - model, type_k, type_v, - v_trans, offload, unified, size_dsa, n_seq_max, n_pad, + model, no_alloc, type_k, type_v, + v_trans, offload, unified, size_dsa, n_seq_max, 0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse); LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); kv_swa = std::make_unique( - model, hparams, type_k, type_v, - v_trans, offload, unified, size_swa, n_seq_max, n_pad, + model, no_alloc, hparams, type_k, type_v, + v_trans, offload, unified, size_swa, n_seq_max, hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr); } diff --git a/src/llama-kv-cache-dsa-iswa.h b/src/llama-kv-cache-dsa-iswa.h index 28cf95bf05..73fbaa7dfa 100644 --- a/src/llama-kv-cache-dsa-iswa.h +++ b/src/llama-kv-cache-dsa-iswa.h @@ -14,6 +14,7 @@ class llama_kv_cache_dsa_iswa : public llama_memory_i { public: llama_kv_cache_dsa_iswa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -23,7 +24,6 @@ public: uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, const layer_filter_cb & filter_mla, const layer_filter_cb & filter_lid, const layer_reuse_cb & reuse); diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp index 6e5f7d4840..2602ae3c31 100644 --- a/src/llama-kv-cache-dsa.cpp +++ b/src/llama-kv-cache-dsa.cpp @@ -13,6 +13,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -20,7 +21,6 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, const layer_filter_cb & filter_mla, @@ -31,8 +31,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size); kv_mla = std::make_unique( - model, model.hparams, type_k, type_v, - v_trans, offload, unified, kv_size, n_seq_max, n_pad, + model, no_alloc, model.hparams, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_swa, swa_type, nullptr, filter_mla, reuse, nullptr); // we use llama_kv_cache for caching indexer keys @@ -48,8 +48,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); kv_lid = std::make_unique( - model, hparams_lid, type_k, type_v, - v_trans, offload, unified, kv_size, n_seq_max, n_pad, + model, no_alloc, hparams_lid, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_swa, swa_type, nullptr, filter_lid, reuse, nullptr); } diff --git a/src/llama-kv-cache-dsa.h b/src/llama-kv-cache-dsa.h index e74fc4d910..13a59047cc 100644 --- a/src/llama-kv-cache-dsa.h +++ b/src/llama-kv-cache-dsa.h @@ -16,6 +16,7 @@ class llama_kv_cache_dsa : public llama_memory_i { public: llama_kv_cache_dsa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -23,7 +24,6 @@ public: bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, const layer_filter_cb & filter_mla, diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 6186f9d34e..f0ab9f35dc 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -931,21 +931,24 @@ static void dsv4_make_k_only(llama_hparams & hparams) { llama_dsv4_comp_state::llama_dsv4_comp_state( const llama_model & model, - bool offload, - bool unified, - uint32_t n_seq_max, - uint32_t ratio, - uint32_t state_size, - uint32_t n_embd_state, - uint32_t n_rs_seq, - const char * name, + bool no_alloc, + bool offload, + bool unified, + uint32_t n_seq_max, + uint32_t ratio, + uint32_t state_size, + uint32_t n_embd_state, + uint32_t n_rs_seq, + const char * name, const llama_memory_i::layer_filter_cb & filter) : - hparams(model.hparams), ratio(ratio), state_size(state_size), n_embd_state(n_embd_state), n_stream(unified ? 1 : n_seq_max), - n_rs_seq(n_rs_seq) { + n_rs_seq(n_rs_seq), + ctxs_bufs(no_alloc) { + const llama_hparams & hparams = model.hparams; + struct ggml_backend_buft_comparator { bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const { return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0; @@ -1020,25 +1023,10 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( } for (auto & [buft, ctx] : ctx_map) { - ggml_backend_buffer_t buf; - if (hparams.no_alloc) { - buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { - t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it - } - } else { - buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer - } - if (!buf) { - throw std::runtime_error("failed to allocate buffer for DSV4 compressor state"); - } - - ggml_backend_buffer_clear(buf, 0); + ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "DSV4 compressor state"); LLAMA_LOG_INFO("%s: %10s DSV4 %s state buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), name, ggml_backend_buffer_get_size(buf)/1024.0/1024.0); - - ctxs_bufs.emplace_back(std::move(ctx), buf); } LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n", @@ -1119,18 +1107,7 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const { } std::map llama_dsv4_comp_state::memory_breakdown() const { - std::map ret; - for (const auto & [ctx, buf] : ctxs_bufs) { - ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); - - if (hparams.no_alloc) { - GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); - ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); - } else { - ret[buft] += ggml_backend_buffer_get_size(buf.get()); - } - } - return ret; + return ctxs_bufs.memory_breakdown(); } void llama_dsv4_comp_state::state_write( @@ -1265,6 +1242,7 @@ size_t llama_dsv4_comp_state::total_size() const { llama_kv_cache_dsv4::llama_kv_cache_dsv4( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -1274,7 +1252,6 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, uint32_t n_rs_seq, const layer_filter_cb & filter, const layer_reuse_cb & reuse) : @@ -1309,8 +1286,8 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( dsv4_make_k_only(hparams_raw); kv_raw = std::make_unique( - model, hparams_raw, type_k, type_v, - v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad, + model, no_alloc, hparams_raw, type_k, type_v, + v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, nullptr, filter_raw, reuse, nullptr); // V4.1: CSA slot = ratio 2, HCA slot = ratio 1, no overlap and no indexer compressor; the index K is stored @@ -1413,24 +1390,24 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( __func__, dsv4_comp_size(kv_size, ratio_csa)); kv_csa = std::make_unique( - model, hparams_csa, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad, + model, no_alloc, hparams_csa, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr); LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n", __func__, dsv4_comp_size(kv_size, ratio_hca)); kv_hca = std::make_unique( - model, hparams_hca, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad, + model, no_alloc, hparams_hca, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr); LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n", __func__, dsv4_comp_size(kv_size, ratio_csa)); kv_lid = std::make_unique( - model, hparams_lid, type_k, type_v, - v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad, + model, no_alloc, hparams_lid, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr); LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__); @@ -1439,19 +1416,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k(); csa_state = std::make_unique( - model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size, + model, no_alloc, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size, csa_state_embd, n_rs_seq, "csa", filter_csa_state); LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__); hca_state = std::make_unique( - model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca, + model, no_alloc, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca, model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca_state); LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__); lid_state = std::make_unique( - model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, + model, no_alloc, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid); // DSV4 attention reads compressed-K / compressor-state rows that the current diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index bfc52947fd..44bf959554 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -16,14 +16,15 @@ public: llama_dsv4_comp_state( const llama_model & model, - bool offload, - bool unified, - uint32_t n_seq_max, - uint32_t ratio, - uint32_t state_size, - uint32_t n_embd_state, - uint32_t n_rs_seq, - const char * name, + bool no_alloc, + bool offload, + bool unified, + uint32_t n_seq_max, + uint32_t ratio, + uint32_t state_size, + uint32_t n_embd_state, + uint32_t n_rs_seq, + const char * name, const llama_memory_i::layer_filter_cb & filter); void clear(llama_seq_id seq_id, bool data); @@ -61,15 +62,13 @@ private: std::vector score_stream; }; - const llama_hparams & hparams; - const uint32_t ratio; const uint32_t state_size; const uint32_t n_embd_state; const uint32_t n_stream; const uint32_t n_rs_seq; - std::vector> ctxs_bufs; + llama_memory_bufs ctxs_bufs; std::vector layers; @@ -92,6 +91,7 @@ class llama_kv_cache_dsv4 : public llama_memory_i { public: llama_kv_cache_dsv4( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -101,7 +101,6 @@ public: uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, uint32_t n_rs_seq, const layer_filter_cb & filter, const layer_reuse_cb & reuse); diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cpp index da440f6a2d..8ce8e0165f 100644 --- a/src/llama-kv-cache-iswa.cpp +++ b/src/llama-kv-cache-iswa.cpp @@ -13,6 +13,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -22,18 +23,18 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, const layer_share_cb & share) : - llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified, - kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) { + llama_kv_cache_iswa(model, no_alloc, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified, + kv_size, n_seq_max, n_ubatch, mem_other, filter, reuse, share) { } llama_kv_cache_iswa::llama_kv_cache_iswa( const llama_model & model, - const llama_hparams & hparams, + bool no_alloc, + const llama_hparams & hparams, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -43,7 +44,6 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, @@ -93,15 +93,15 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( } kv_base = std::make_unique( - model, hparams, type_k, type_v, - v_trans, offload, unified, size_base, n_seq_max, n_pad, + model, no_alloc, hparams, type_k, type_v, + v_trans, offload, unified, size_base, n_seq_max, 0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share); LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); kv_swa = std::make_unique( - model, hparams, type_k, type_v, - v_trans, offload, unified, size_swa, n_seq_max, n_pad, + model, no_alloc, hparams, type_k, type_v, + v_trans, offload, unified, size_swa, n_seq_max, hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share); } diff --git a/src/llama-kv-cache-iswa.h b/src/llama-kv-cache-iswa.h index 7dab6eaa82..65aba2af7a 100644 --- a/src/llama-kv-cache-iswa.h +++ b/src/llama-kv-cache-iswa.h @@ -15,6 +15,7 @@ class llama_kv_cache_iswa : public llama_memory_i { public: llama_kv_cache_iswa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -24,7 +25,6 @@ public: uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, @@ -32,7 +32,8 @@ public: llama_kv_cache_iswa( const llama_model & model, - const llama_hparams & hparams, + bool no_alloc, + const llama_hparams & hparams, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -42,7 +43,6 @@ public: uint32_t kv_size, uint32_t n_seq_max, uint32_t n_ubatch, - uint32_t n_pad, llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, diff --git a/src/llama-kv-cache-msa.cpp b/src/llama-kv-cache-msa.cpp index 55ef286caf..2d6bac0cb2 100644 --- a/src/llama-kv-cache-msa.cpp +++ b/src/llama-kv-cache-msa.cpp @@ -12,6 +12,7 @@ llama_kv_cache_msa::llama_kv_cache_msa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -19,21 +20,20 @@ llama_kv_cache_msa::llama_kv_cache_msa( bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, const layer_filter_cb & filter, const layer_filter_cb & filter_idx, const layer_reuse_cb & reuse) : hparams_idx(model.hparams), - n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_pad(n_pad), + n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_swa(n_swa), swa_type(swa_type) { LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size); kv_base = std::make_unique( - model, model.hparams, type_k, type_v, - v_trans, offload, unified, kv_size, n_seq_max, n_pad, + model, no_alloc, model.hparams, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_swa, swa_type, nullptr, filter, reuse, nullptr); // the MSA indexer uses a single key head per layer @@ -44,8 +44,8 @@ llama_kv_cache_msa::llama_kv_cache_msa( LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); kv_idx = std::make_unique( - model, hparams_idx, type_k, type_v, - v_trans, offload, unified, kv_size, n_seq_max, n_pad, + model, no_alloc, hparams_idx, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_swa, swa_type, nullptr, filter_idx, reuse, nullptr); } @@ -261,7 +261,7 @@ const llama_kv_cache_context * llama_kv_cache_msa_context::get_idx() const { uint32_t llama_kv_cache_msa_context::get_n_pos() const { // pad the value so that the graph remains constant across batches and can be reused - const uint32_t n_pad_cur = std::max(kv->get_n_pad(), 256u); + const uint32_t n_pad = 256; llama_pos pos_max = -1; @@ -269,7 +269,7 @@ uint32_t llama_kv_cache_msa_context::get_n_pos() const { pos_max = std::max(pos_max, kv->seq_pos_max(seq_id)); } - return std::max(n_pad_cur, GGML_PAD((uint32_t) (pos_max + 1), n_pad_cur)); + return std::max(n_pad, GGML_PAD((uint32_t) (pos_max + 1), n_pad)); } void llama_kv_cache_msa_context::set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const { diff --git a/src/llama-kv-cache-msa.h b/src/llama-kv-cache-msa.h index f09b6d32b0..81c6827f3e 100644 --- a/src/llama-kv-cache-msa.h +++ b/src/llama-kv-cache-msa.h @@ -15,6 +15,7 @@ class llama_kv_cache_msa : public llama_memory_i { public: llama_kv_cache_msa( const llama_model & model, + bool no_alloc, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -22,7 +23,6 @@ public: bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, const layer_filter_cb & filter, @@ -67,7 +67,6 @@ public: llama_kv_cache * get_base() const; llama_kv_cache * get_idx () const; - uint32_t get_n_pad() const { return n_pad; } uint32_t get_n_seq_max() const { return n_seq_max; } uint32_t get_n_swa() const { return n_swa; } llama_swa_type get_swa_type() const { return swa_type; } @@ -78,7 +77,6 @@ private: const uint32_t n_stream = 1; const uint32_t n_seq_max = 1; - const uint32_t n_pad = 1; const uint32_t n_swa = 0; const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 4ab280a619..e108f0423b 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -64,7 +64,8 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) { llama_kv_cache::llama_kv_cache( const llama_model & model, - const llama_hparams & hparams, + bool no_alloc, + const llama_hparams & hparams, ggml_type type_k, ggml_type type_v, bool v_trans, @@ -72,16 +73,16 @@ llama_kv_cache::llama_kv_cache( bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, const layer_share_cb & share, - const char * name_tag) : + const char * name_tag) : model(model), hparams(hparams), v_trans(v_trans), - n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), swa_type(swa_type), + ctxs_bufs(no_alloc), other(static_cast(mem_other)), v_cells_impl(other ? other->v_cells_impl : std::make_shared()), v_cells(*v_cells_impl) { @@ -97,8 +98,6 @@ llama_kv_cache::llama_kv_cache( } } - GGML_ASSERT(kv_size % n_pad == 0); - const uint32_t n_layer = hparams.n_layer_all; // define a comparator for the buft -> ctx map to ensure that the order is well-defined: @@ -273,25 +272,11 @@ llama_kv_cache::llama_kv_cache( } } - // allocate tensors and initialize the buffers to avoid NaNs in the padding + // allocate the tensors and initialize the buffers to avoid NaNs in the padding for (auto & [buft, ctx] : ctx_map) { - ggml_backend_buffer_t buf; - if (hparams.no_alloc) { - buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { - t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it - } - } else { - buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer - } - if (!buf) { - throw std::runtime_error("failed to allocate buffer for kv cache"); - } + ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "kv cache"); LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); - - ggml_backend_buffer_clear(buf, 0); - ctxs_bufs.emplace_back(std::move(ctx), buf); } { @@ -705,20 +690,7 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const { } std::map llama_kv_cache::memory_breakdown() const { - std::map ret; - for (const auto & [ctx, buf] : ctxs_bufs) { - ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); - - if (hparams.no_alloc) { - GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); - ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); - } else { - // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base - ret[buft] += ggml_backend_buffer_get_size(buf.get()); - } - } - - return ret; + return ctxs_bufs.memory_breakdown(); } llama_memory_context_ptr llama_kv_cache::init_batch( @@ -1285,12 +1257,12 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { // pad the n_kv value so that the graph remains constant across batches and can be reused // note: this also helps some backends with performance (f.ex https://github.com/ggml-org/llama.cpp/pull/16812#issuecomment-3455112220) - const uint32_t n_pad_cur = std::max(n_pad, 256u); + const uint32_t n_pad = 256; for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { const auto & cells = v_cells[sinfo.strm[s]]; - result = std::max(std::min(cells.size(), std::max(n_pad_cur, GGML_PAD(cells.used_max_p1(), n_pad_cur))), result); + result = std::max(std::min(cells.size(), std::max(n_pad, GGML_PAD(cells.used_max_p1(), n_pad))), result); } return result; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 9bfb8f231f..60d2e183c6 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -98,6 +98,7 @@ public: // likely through `struct llama_memory_params` llama_kv_cache( const llama_model & model, + bool no_alloc, const llama_hparams & hparams, ggml_type type_k, ggml_type type_v, @@ -106,7 +107,6 @@ public: bool unified, uint32_t kv_size, uint32_t n_seq_max, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, llama_memory_t mem_other, @@ -271,9 +271,6 @@ private: const uint32_t n_seq_max = 1; const uint32_t n_stream = 1; - // required padding - const uint32_t n_pad = 1; - // SWA const uint32_t n_swa = 0; @@ -296,7 +293,7 @@ private: const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; // ggml contexts for the KV cache along with the allocated backend buffers: - std::vector> ctxs_bufs; + llama_memory_bufs ctxs_bufs; // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot()) // note: this is not part of the KV state and it's only used to speed-up the find_slot() method diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 8113f5c21d..da17476494 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -1,20 +1,16 @@ #include "llama-memory-hybrid-idx.h" -#include -#include -#include - #include "llama-impl.h" #include "llama-batch.h" #include "llama-io.h" #include "llama-model.h" - #include #include #include #include #include +#include // // llama_memory_hybrid_idx @@ -22,12 +18,12 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, bool v_trans, uint32_t kv_size, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, /* recurrent */ @@ -45,7 +41,8 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( const layer_filter_cb & filter_idx) : llama_memory_hybrid( model, - type_k, type_v, v_trans, kv_size, n_pad, n_swa, swa_type, + no_alloc, + type_k, type_v, v_trans, kv_size, n_swa, swa_type, type_r, type_s, rs_size, n_seq_max, n_rs_seq, offload, unified, filter_attn, filter_recr), @@ -68,8 +65,8 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); return new llama_kv_cache( - model, hparams_idx, type_k, type_v, v_trans, offload, unified, - kv_size, n_seq_max, n_pad, n_swa, swa_type, + model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified, + kv_size, n_seq_max, n_swa, swa_type, nullptr, filter_idx, nullptr, nullptr, "idx_"); }()) {} diff --git a/src/llama-memory-hybrid-idx.h b/src/llama-memory-hybrid-idx.h index 0ac4265dfc..59940dbb84 100644 --- a/src/llama-memory-hybrid-idx.h +++ b/src/llama-memory-hybrid-idx.h @@ -18,12 +18,12 @@ class llama_memory_hybrid_idx : public llama_memory_hybrid { public: llama_memory_hybrid_idx( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, bool v_trans, uint32_t kv_size, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, /* recurrent */ diff --git a/src/llama-memory-hybrid-iswa.cpp b/src/llama-memory-hybrid-iswa.cpp index 06f7fd5428..6d843c76ee 100644 --- a/src/llama-memory-hybrid-iswa.cpp +++ b/src/llama-memory-hybrid-iswa.cpp @@ -10,6 +10,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, @@ -17,7 +18,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( bool swa_full, uint32_t kv_size, uint32_t n_ubatch, - uint32_t n_pad, /* recurrent */ ggml_type type_r, ggml_type type_s, @@ -33,6 +33,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( hparams(model.hparams), mem_attn(new llama_kv_cache_iswa( model, + no_alloc, type_k, type_v, v_trans, @@ -42,7 +43,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( kv_size, n_seq_max, n_ubatch, - n_pad, nullptr, filter_attn == nullptr ? [&](int32_t il) { return !hparams.is_recr(il); } @@ -52,6 +52,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( )), mem_recr(new llama_memory_recurrent( model, + no_alloc, type_r, type_s, offload, diff --git a/src/llama-memory-hybrid-iswa.h b/src/llama-memory-hybrid-iswa.h index c9d3f9f57c..52c885ac93 100644 --- a/src/llama-memory-hybrid-iswa.h +++ b/src/llama-memory-hybrid-iswa.h @@ -20,6 +20,7 @@ class llama_memory_hybrid_iswa : public llama_memory_i { public: llama_memory_hybrid_iswa( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, @@ -27,7 +28,6 @@ public: bool swa_full, uint32_t kv_size, uint32_t n_ubatch, - uint32_t n_pad, /* recurrent */ ggml_type type_r, ggml_type type_s, diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp index bbd4bee0b5..7d3b279088 100644 --- a/src/llama-memory-hybrid.cpp +++ b/src/llama-memory-hybrid.cpp @@ -10,12 +10,12 @@ llama_memory_hybrid::llama_memory_hybrid( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, bool v_trans, uint32_t kv_size, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, /* recurrent */ @@ -33,6 +33,7 @@ llama_memory_hybrid::llama_memory_hybrid( hparams(model.hparams), mem_attn(new llama_kv_cache( model, + no_alloc, model.hparams, type_k, type_v, @@ -41,7 +42,6 @@ llama_memory_hybrid::llama_memory_hybrid( unified, kv_size, n_seq_max, - n_pad, n_swa, swa_type, nullptr, @@ -53,6 +53,7 @@ llama_memory_hybrid::llama_memory_hybrid( )), mem_recr(new llama_memory_recurrent( model, + no_alloc, type_r, type_s, offload, diff --git a/src/llama-memory-hybrid.h b/src/llama-memory-hybrid.h index 484eafb749..2b4d1269c3 100644 --- a/src/llama-memory-hybrid.h +++ b/src/llama-memory-hybrid.h @@ -20,12 +20,12 @@ class llama_memory_hybrid : public llama_memory_i { public: llama_memory_hybrid( const llama_model & model, + bool no_alloc, /* attn */ ggml_type type_k, ggml_type type_v, bool v_trans, uint32_t kv_size, - uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, /* recurrent */ diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index d00630d4c5..5fd644645c 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -19,13 +19,14 @@ llama_memory_recurrent::llama_memory_recurrent( const llama_model & model, + bool no_alloc, ggml_type type_r, ggml_type type_s, bool offload, uint32_t mem_size, uint32_t n_seq_max, uint32_t n_rs_seq, - const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) { + const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max), ctxs_bufs(no_alloc) { const int32_t n_layer = hparams.n_layer(); head = 0; @@ -114,23 +115,11 @@ llama_memory_recurrent::llama_memory_recurrent( } } - // allocate tensors and initialize the buffers to avoid NaNs in the padding + // allocate the tensors and initialize the buffers to avoid NaNs in the padding for (auto & [buft, ctx] : ctx_map) { - ggml_backend_buffer_t buf; - if (hparams.no_alloc) { - buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { - t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it - } - } else { - buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer - } - if (!buf) { - throw std::runtime_error("failed to allocate buffer for rs cache"); - } - ggml_backend_buffer_clear(buf, 0); + ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "rs cache"); + LLAMA_LOG_INFO("%s: %10s RS buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); - ctxs_bufs.emplace_back(std::move(ctx), buf); } if (is_empty()) { @@ -442,18 +431,7 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) { } std::map llama_memory_recurrent::memory_breakdown() const { - std::map ret; - for (const auto & [ctx, buf] : ctxs_bufs) { - ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); - - if (hparams.no_alloc) { - GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); - ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); - } else { - ret[buft] += ggml_backend_buffer_get_size(buf.get()); - } - } - return ret; + return ctxs_bufs.memory_breakdown(); } llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index 08489a4bee..d0fc14779c 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -18,6 +18,7 @@ class llama_memory_recurrent : public llama_memory_i { public: llama_memory_recurrent( const llama_model & model, + bool no_alloc, ggml_type type_r, ggml_type type_s, bool offload, @@ -120,8 +121,8 @@ private: const uint32_t n_seq_max = 1; - // ggml contexts for the KV cache along with the allocated backend buffers: - std::vector> ctxs_bufs; + // ggml contexts for the RS cache along with the allocated backend buffers: + llama_memory_bufs ctxs_bufs; // true if no layers - can happen if the layer filter removes all layers bool is_empty() const; diff --git a/src/llama-memory.cpp b/src/llama-memory.cpp index 961dd74ef5..042aab6082 100644 --- a/src/llama-memory.cpp +++ b/src/llama-memory.cpp @@ -4,6 +4,8 @@ #include "llama-io.h" #include +#include +#include #include llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) { @@ -64,6 +66,52 @@ bool llama_memory_status_is_fail(llama_memory_status status) { return false; } +// +// llama_memory_bufs +// + +ggml_backend_buffer_ptr llama_memory_bufs::alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name) { + ggml_backend_buffer_ptr buf; + if (no_alloc) { + buf.reset(ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0)); // dummy buffer + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + t->buffer = buf.get(); // set dummy buffer so that the backend scheduler won't try to allocate the tensors + } + } else { + buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft)); // real buffer + } + if (!buf) { + throw std::runtime_error(std::string("failed to allocate buffer for ") + name); + } + return buf; +} + +ggml_backend_buffer_t llama_memory_bufs::emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name) { + ggml_backend_buffer_ptr buf = alloc_buf(ctx.get(), buft, no_alloc, name); + + // initialize the buffer to avoid NaNs in the padding + ggml_backend_buffer_clear(buf.get(), 0); + + bufs.emplace_back(std::move(ctx), std::move(buf)); + + return bufs.back().second.get(); +} + +std::map llama_memory_bufs::memory_breakdown() const { + std::map ret; + for (const auto & [ctx, buf] : bufs) { + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); + + if (no_alloc) { + GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); + ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); + } else { + ret[buft] += ggml_backend_buffer_get_size(buf.get()); + } + } + return ret; +} + // collects the buffers of the tensors that a state write visits, without keeping the data struct llama_memory_buffers_writer : public llama_io_write_i { std::set buffers; @@ -84,6 +132,10 @@ struct llama_memory_buffers_writer : public llama_io_write_i { }; std::vector llama_memory_get_buffers(const llama_memory_t mem) { + if (mem == nullptr) { + return {}; + } + llama_memory_buffers_writer writer; mem->state_write(writer); diff --git a/src/llama-memory.h b/src/llama-memory.h index b33c7aee41..37d01ed116 100644 --- a/src/llama-memory.h +++ b/src/llama-memory.h @@ -6,6 +6,7 @@ #include #include #include +#include struct llama_ubatch; @@ -25,6 +26,36 @@ struct llama_memory_params { llama_context_type ctx_type; llama_memory_t mem_other; + + // only simulate the memory allocations + bool no_alloc; +}; + +// the tensor contexts and backend buffers of a memory instance +// with no_alloc, dummy buffers are attached instead of allocating the tensor data, so the memory use can still be measured +struct llama_memory_bufs { + explicit llama_memory_bufs(bool no_alloc) : no_alloc(no_alloc) {} + + // allocate the tensors of ctx on buft, or attach a dummy buffer if no_alloc is set + // returns the buffer, for logging + ggml_backend_buffer_t emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name); + + // the memory used by the tensors: the buffer sizes, or the simulated sizes if no_alloc is set + std::map memory_breakdown() const; + + // allocate a single ctx buffer - for the memory instances that keep their own buffer layout + static ggml_backend_buffer_ptr alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name); + + bool empty() const { return bufs.empty(); } + + auto begin() { return bufs.begin(); } + auto end() { return bufs.end(); } + auto begin() const { return bufs.begin(); } + auto end() const { return bufs.end(); } + + std::vector> bufs; + + const bool no_alloc; }; enum llama_memory_status { diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 1de65c728d..0220c5ab8f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2448,10 +2448,14 @@ llama_split_mode llama_model::split_mode() const { return params.split_mode; } +bool llama_model::no_alloc() const { + return params.no_alloc; +} + std::map llama_model::memory_breakdown() const { std::map ret; for (const auto & [ctx, bufs] : pimpl->ctxs_bufs) { - if (hparams.no_alloc) { + if (no_alloc()) { GGML_ASSERT(bufs.size() == 1); ggml_backend_buffer_t buf = bufs[0].get(); GGML_ASSERT(ggml_backend_buffer_get_base(buf) == nullptr); @@ -2506,7 +2510,7 @@ void llama_model::print_info() const { // hparams LLAMA_LOG_INFO("%s: arch = %s\n", __func__, arch_name().c_str()); LLAMA_LOG_INFO("%s: vocab_only = %d\n", __func__, hparams.vocab_only); - LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, hparams.no_alloc); + LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, no_alloc()); if (!hparams.vocab_only) { LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train); @@ -2830,6 +2834,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_msa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -2837,7 +2842,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, @@ -2856,6 +2860,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache( *this, + params.no_alloc, hparams, params.type_k, params.type_v, @@ -2864,7 +2869,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, @@ -2882,6 +2886,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_dsa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -2889,7 +2894,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, filter_mla, @@ -2923,11 +2927,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_memory_hybrid_idx( /* model */ *this, + /* no_alloc */ params.no_alloc, /* attn_type_k */ params.type_k, /* attn_type_v */ params.type_v, /* attn_v_trans */ !cparams.flash_attn, /* attn_kv_size */ cparams.n_ctx_seq, - /* attn_n_pad */ 1, /* attn_n_swa */ hparams.n_swa, /* attn_swa_type */ hparams.swa_type, /* recurrent_type_r */ GGML_TYPE_F32, @@ -2947,6 +2951,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, // full-attention checkpoint: no indexer, so no indexer key cache res = new llama_kv_cache( *this, + params.no_alloc, hparams, params.type_k, params.type_v, @@ -2955,7 +2960,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, @@ -2968,6 +2972,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_dsa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -2975,7 +2980,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, @@ -2994,6 +2998,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache( *this, + params.no_alloc, hparams, params.type_k, params.type_v, @@ -3002,7 +3007,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, @@ -3019,6 +3023,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_dsa_iswa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3028,7 +3033,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, filter_mla, filter_lid, nullptr); @@ -3046,6 +3050,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_iswa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3055,7 +3060,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, nullptr, filter_mtp, nullptr, @@ -3063,6 +3067,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } else { res = new llama_kv_cache_dsv4( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3072,7 +3077,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, cparams.n_rs_seq, nullptr, nullptr); @@ -3086,6 +3090,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_iswa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3095,7 +3100,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, nullptr, nullptr, nullptr, @@ -3120,6 +3124,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, if (llm_arch_is_recurrent(arch)) { res = new llama_memory_recurrent( *this, + params.no_alloc, GGML_TYPE_F32, GGML_TYPE_F32, cparams.offload_kqv, @@ -3175,13 +3180,13 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, // Use hybrid-iswa for hybrid models with SWA res = new llama_memory_hybrid_iswa( /* model */ *this, + /* no_alloc */ params.no_alloc, /* attn_type_k */ params.type_k, /* attn_type_v */ params.type_v, /* attn_v_trans */ !cparams.flash_attn, /* attn_swa_full */ params.swa_full, /* attn_kv_size */ cparams.n_ctx_seq, /* attn_n_ubatch */ cparams.n_ubatch, - /* attn_n_pad */ 1, /* recurrent_type_r */ GGML_TYPE_F32, /* recurrent_type_s */ GGML_TYPE_F32, /* recurrent_rs_size */ std::max((uint32_t) 1, cparams.n_seq_max), @@ -3195,11 +3200,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, // sparse attention over a per-token indexer cache, in its own memory type res = new llama_memory_hybrid_idx( /* model */ *this, + /* no_alloc */ params.no_alloc, /* attn_type_k */ params.type_k, /* attn_type_v */ params.type_v, /* attn_v_trans */ !cparams.flash_attn, /* attn_kv_size */ cparams.n_ctx_seq, - /* attn_n_pad */ 1, /* attn_n_swa */ hparams.n_swa, /* attn_swa_type */ hparams.swa_type, /* recurrent_type_k */ GGML_TYPE_F32, @@ -3215,11 +3220,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } else { res = new llama_memory_hybrid( /* model */ *this, + /* no_alloc */ params.no_alloc, /* attn_type_k */ params.type_k, /* attn_type_v */ params.type_v, /* attn_v_trans */ !cparams.flash_attn, /* attn_kv_size */ cparams.n_ctx_seq, - /* attn_n_pad */ 1, /* attn_n_swa */ hparams.n_swa, /* attn_swa_type */ hparams.swa_type, /* recurrent_type_k */ GGML_TYPE_F32, @@ -3281,6 +3286,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache_iswa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3290,7 +3296,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, mem_other, filter, reuse, @@ -3298,6 +3303,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } else { res = new llama_kv_cache_iswa( *this, + params.no_alloc, params.type_k, params.type_v, !cparams.flash_attn, @@ -3307,7 +3313,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_ctx_seq, cparams.n_seq_max, cparams.n_ubatch, - 1, nullptr, filter, reuse, @@ -3318,6 +3323,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = new llama_kv_cache( *this, + params.no_alloc, hparams, params.type_k, params.type_v, @@ -3326,7 +3332,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.kv_unified, cparams.n_ctx_seq, cparams.n_seq_max, - 1, hparams.n_swa, hparams.swa_type, nullptr, diff --git a/src/llama-model.h b/src/llama-model.h index ccc2602e5f..222238efed 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -886,6 +886,7 @@ struct llama_model { uint32_t n_gpu_layers() const; llama_split_mode split_mode() const; + bool no_alloc() const; std::map memory_breakdown() const; diff --git a/src/llama-moe-cache.cpp b/src/llama-moe-cache.cpp index 4d3596b244..6a343a7603 100644 --- a/src/llama-moe-cache.cpp +++ b/src/llama-moe-cache.cpp @@ -1,6 +1,7 @@ #include "llama-moe-cache.h" #include "llama-impl.h" +#include "llama-memory.h" #include "llama-model.h" #include "ggml-cpp.h" @@ -252,7 +253,7 @@ struct llama_moe_cache::impl { // views used by copy_experts ggml_context_ptr ctx_views; - impl(const llama_model & model, const std::vector & backends, const std::vector & bufts, size_t size) : + impl(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size) : n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) { for (size_t i = 0; i < backends.size(); ++i) { const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i])); @@ -418,37 +419,19 @@ struct llama_moe_cache::impl { devices[g.id].buf_size += alloc_size(g, g.n_slots); } - if (model.hparams.no_alloc) { - // only used to measure the memory use, see llama_context::memory_breakdown - for (device & d : devices) { - if (!d.ctx) { - continue; - } - d.buf.reset(ggml_backend_buft_alloc_buffer(d.buft, 0)); - for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) { - t->buffer = d.buf.get(); - } + // with no_alloc, dummy buffers are attached so the memory use can still be measured, see llama_context::memory_breakdown + for (device & d : devices) { + if (!d.ctx) { + continue; } - buf_host.reset(ggml_backend_buft_alloc_buffer(buft_host, 0)); - for (ggml_tensor * t = ggml_get_first_tensor(ctx_host.get()); t != nullptr; t = ggml_get_next_tensor(ctx_host.get(), t)) { - t->buffer = buf_host.get(); - } - } else { - for (device & d : devices) { - if (!d.ctx) { - continue; - } - d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft)); - if (!d.buf) { - throw std::runtime_error("failed to allocate the MoE cache buffers"); - } + d.buf = llama_memory_bufs::alloc_buf(d.ctx.get(), d.buft, no_alloc, "MoE cache"); + if (!no_alloc) { ggml_backend_buffer_clear(d.buf.get(), 0); d.buf_size = ggml_backend_buffer_get_size(d.buf.get()); } - buf_host.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_host.get(), buft_host)); - if (!buf_host) { - throw std::runtime_error("failed to allocate the MoE cache buffers"); - } + } + buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps"); + if (!no_alloc) { ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1 buf_host_size = ggml_backend_buffer_get_size(buf_host.get()); @@ -633,8 +616,8 @@ struct llama_moe_cache::impl { } }; -llama_moe_cache::llama_moe_cache(const llama_model & model, const std::vector & backends, const std::vector & bufts, size_t size) : - pimpl(new impl(model, backends, bufts, size)) { +llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size) : + pimpl(new impl(model, no_alloc, backends, bufts, size)) { } llama_moe_cache::~llama_moe_cache() = default; diff --git a/src/llama-moe-cache.h b/src/llama-moe-cache.h index 9baf063909..6c8e36582a 100644 --- a/src/llama-moe-cache.h +++ b/src/llama-moe-cache.h @@ -13,7 +13,7 @@ struct llama_model; class llama_moe_cache { public: // backends are all the backends of the context, each GPU gets its own cache of the given size for the layers assigned to it - llama_moe_cache(const llama_model & model, const std::vector & backends, const std::vector & bufts, size_t size); + llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size); ~llama_moe_cache(); // the device that caches layer il diff --git a/src/llama.cpp b/src/llama.cpp index 9beb14841f..e3699ea1d5 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -342,7 +342,6 @@ static std::pair llama_model_load(struct gguf_context * meta model->t_start_us = tm.t_start_us; model->hparams.vocab_only = params.vocab_only; - model->hparams.no_alloc = params.no_alloc; try { model->load_hparams(ml); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 13f6a5accb..4d58b2fd27 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -342,24 +342,23 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn( return comb; } -ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( +llama_model_deepseek4::graph::hc_pre +llama_model_deepseek4::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { - return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il); + hc_mixes mixes = build_hc_mixes(x, hc_fn, hc_scale, hc_base, il); + return { build_hc_pre(x, mixes.pre, il), mixes.post, mixes.comb }; } -ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( +llama_model_deepseek4::graph::hc_mixes +llama_model_deepseek4::graph::build_hc_mixes( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc*n_embd; @@ -386,28 +385,29 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); cb(pre, "hc_pre", il); - *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); - *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post); - *post = ggml_sigmoid(ctx0, *post); - *post = ggml_scale(ctx0, *post, 2.0f); - cb(*post, "hc_post", il); + ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); + post = dsv4_hc_affine(ctx0, post, scale_post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, 2.0f); + cb(post, "hc_post", il); + ggml_tensor * comb = nullptr; if (cparams.fused_dsv4_hc_comb) { - *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, (int32_t) hparams.dsv4_hc_sinkhorn_iters); - res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il}); } else { ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); - *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); - *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); - *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); - *comb = build_hc_sinkhorn(*comb, il); + comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb); + comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt); + comb = build_hc_sinkhorn(comb, il); } - cb(*comb, "hc_comb", il); + cb(comb, "hc_comb", il); - return pre; + return { pre, post, comb }; } ggml_tensor * llama_model_deepseek4::graph::build_hc_post( @@ -1252,14 +1252,12 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p } ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - cur = build_hc_pre(inpL, + const hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, model.layers[il].hc_attn_base, - &post, &comb, il); + il); + cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1267,20 +1265,21 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cur = build_attention(model, inp_dsv4, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + const hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, model.layers[il].hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); ggml_build_forward_expand(gf, residual); - ggml_build_forward_expand(gf, post); - ggml_build_forward_expand(gf, comb); + ggml_build_forward_expand(gf, ffn_hc.post); + ggml_build_forward_expand(gf, ffn_hc.comb); cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -1330,7 +1329,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_last", il); } @@ -1430,14 +1429,12 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cb(inpL, "mtp_eh_proj", il); ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * cur = build_hc_pre(inpL, + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, - &post, &comb, il); + il); + ggml_tensor * cur = attn_hc.cur; cb(cur, "mtp_hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1445,15 +1442,16 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_attention(model, inp_attn, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "mtp_hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "mtp_hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1483,7 +1481,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "mtp_ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "mtp_l_out", il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 6a9003d42b..97b2cbabef 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -847,10 +847,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g } ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); // the first layer's attention collapses with the identity pre-mix: copy 0 cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0)); @@ -861,18 +858,19 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + pre_prev = ffn_mix.pre; - cur = build_hc_pre(inpL, attn_pre, il); + cur = build_hc_pre(inpL, attn_mix.pre, il); cb(cur, "hc_ffn_pre", il); ggml_build_forward_expand(gf, residual); - ggml_build_forward_expand(gf, post); - ggml_build_forward_expand(gf, comb); + ggml_build_forward_expand(gf, ffn_mix.post); + ggml_build_forward_expand(gf, ffn_mix.comb); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -912,7 +910,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_out", il); } diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 6bbb534f08..e3c2f1f295 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -1204,14 +1204,12 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * cur = build_hc_pre(inpL, + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, - &post, &comb, il); + il); + ggml_tensor * cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1219,15 +1217,16 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ cur = build_attention(model, inp_attn, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1256,7 +1255,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); cb(inpL, "l_out", il); } @@ -1403,10 +1402,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); ggml_tensor * cur; if (pre_prev == nullptr) { @@ -1461,13 +1457,14 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll cb(cur, "attn_out", il); } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + pre_prev = ffn_mix.pre; - cur = build_hc_pre(inpL, attn_pre, il); + cur = build_hc_pre(inpL, attn_mix.pre, il); cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1496,13 +1493,11 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il); ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); cb(l_out, "l_out", il); ggml_build_forward_expand(gf, l_out); - - pre_prev = ffn_pre; } // no hc_head: collapse with the last FFN's pre-mix diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp index 3955609907..d1e9b503d4 100644 --- a/src/models/glm5-next.cpp +++ b/src/models/glm5-next.cpp @@ -442,13 +442,12 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_sinkhorn( return comb; } -ggml_tensor * llama_model_glm5_next::graph::build_hc_pre( +llama_model_glm5_next::graph::hc_pre +llama_model_glm5_next::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc*n_embd; @@ -475,29 +474,29 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_pre( pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); cb(pre, "hc_pre", il); - *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); - *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post); - *post = ggml_sigmoid(ctx0, *post); - *post = ggml_scale(ctx0, *post, 2.0f); - cb(*post, "hc_post", il); + ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); + post = dsv4_hc_affine(ctx0, post, scale_post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, 2.0f); + cb(post, "hc_post", il); + ggml_tensor * comb = nullptr; if (cparams.fused_dsv4_hc_comb) { - *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, (int32_t) hparams.dsv4_hc_sinkhorn_iters); - res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il}); } else { ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); - *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); - *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); - *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); - *comb = build_hc_sinkhorn(*comb, il); + comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb); + comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt); + comb = build_hc_sinkhorn(comb, il); } - cb(*comb, "hc_comb", il); + cb(comb, "hc_comb", il); - ggml_tensor * result = build_hc_pre(x, pre, il); - return result; + return { build_hc_pre(x, pre, il), post, comb }; } ggml_tensor * llama_model_glm5_next::graph::build_hc_post( @@ -609,8 +608,9 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * prev_sel = nullptr; - cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il); + // the NextN block has a full indexer, nothing to carry in + dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, nullptr, il); + cur = dsa.cur; cb(cur, "mtp_attn_out", il); // narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows @@ -711,10 +711,8 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - cur = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); + cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -725,14 +723,17 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda, d_inner, n_seq_tokens, n_seqs, il); } else { - cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il); + dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, prev_sel, il); + cur = dsa.cur; + prev_sel = dsa.sel; } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -770,7 +771,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p cb(cur, "ffn_out", il); } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_out", il); } @@ -1039,10 +1040,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select( // Nope MLA layer with sparse attention over the indexer selection -ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( +llama_model_glm5_next::graph::dsa_out +llama_model_glm5_next::graph::build_dsa_layer( ggml_tensor * cur, const llama_layer & layer, const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, - llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il) { + llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il) { const auto * mctx_mla = mctx_hyb->get_attn(); @@ -1073,13 +1075,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( ggml_tensor * kq_mask = inp_attn->get_kq_mask(); - ggml_tensor * sel = nullptr; + ggml_tensor * sel = prev_sel; if (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il)) { // the NextN block always has a full indexer sel = build_kpool_select(cur, qr, kq_mask, layer, mctx_hyb, inp_kpool, il); - *prev_sel = sel; } else { - GGML_ASSERT(*prev_sel != nullptr && "shared indexer layer must follow a full indexer layer"); - sel = *prev_sel; + GGML_ASSERT(sel != nullptr && "shared indexer layer must follow a full indexer layer"); } ggml_build_forward_expand(gf, q_absorbed); @@ -1099,5 +1099,5 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( out = ggml_mul_mat(ctx0, layer.wo, out); cb(out, "attn_out", il); - return out; + return { out, sel }; } diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 25e40abf7b..3d456fe858 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -172,12 +172,12 @@ static ggml_tensor * hy_v4_hc_reduce(ggml_context * ctx0, ggml_tensor * x, ggml_ return ggml_cast(ctx0, result, out_type); } -ggml_tensor * llama_model_hy_v4::graph::build_hc_pre( +llama_model_hy_v4::graph::hc_pre +llama_model_hy_v4::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t nt = x->ne[2]; @@ -202,16 +202,15 @@ ggml_tensor * llama_model_hy_v4::graph::build_hc_pre( cb(pre, "hc_pre", il); // post = magnitude*sigmoid(mixes[hc:2hc]*scale_post + base_post) + eps - ggml_tensor * po = hy_v4_view_2d(ctx0, mixes, hc, nt, hc); - po = ggml_mul(ctx0, po, scale_post); - po = ggml_add(ctx0, po, base_post); - po = ggml_sigmoid(ctx0, po); - po = ggml_scale(ctx0, po, hparams.hc_magnitude); - po = ggml_scale_bias(ctx0, po, 1.0f, hparams.dsv4_hc_eps); - *post = po; - cb(po, "hc_post_gate", il); + ggml_tensor * post = hy_v4_view_2d(ctx0, mixes, hc, nt, hc); + post = ggml_mul(ctx0, post, scale_post); + post = ggml_add(ctx0, post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, hparams.hc_magnitude); + post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps); + cb(post, "hc_post_gate", il); - return hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type); + return { hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type), post }; } ggml_tensor * llama_model_hy_v4::graph::build_hc_post( @@ -418,12 +417,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( return ggml_cont(ctx0, ggml_top_k(ctx0, score, n_top_k)); } -ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( +llama_model_hy_v4::graph::attn_dsa +llama_model_hy_v4::graph::build_attention_dsa( const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, - ggml_tensor ** last_top_k, + ggml_tensor * last_top_k, float kq_scale, int il) const { const auto & layer = model.layers[il]; @@ -436,11 +436,12 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur); qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * top_k = last_top_k; if (hparams.is_indexer_full(il)) { - *last_top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il); - cb(*last_top_k, "top_k", il); + top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il); + cb(top_k, "top_k", il); } - GGML_ASSERT(*last_top_k != nullptr); + GGML_ASSERT(top_k != nullptr); ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr); @@ -480,7 +481,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * attn = build_attn(inp_attn_dsa, nullptr, nullptr, nullptr, - Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, *last_top_k, kq_scale, il); + Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, top_k, kq_scale, il); cb(attn, "attn_kqv", il); ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur); @@ -491,7 +492,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * out = build_lora_mm(layer.wo, attn); cb(out, "attn_out", il); - return out; + return { out, top_k }; } llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_params & params) : @@ -520,23 +521,27 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param for (int il = 0; il < n_layer; ++il) { ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - - cur = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, - model.layers[il].hc_attn_base, &post, il); + hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, + model.layers[il].hc_attn_base, il); + cur = attn_hc.cur; cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - cur = is_dsa - ? build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, &last_top_k, kq_scale, il) - : build_attention(model, inp_attn, cur, inp_pos, kq_scale, il); + if (is_dsa) { + attn_dsa attn = build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il); + cur = attn.cur; + last_top_k = attn.top_k; + } else { + cur = build_attention(model, inp_attn, cur, inp_pos, kq_scale, il); + } - inpL = build_hc_post(cur, residual, post, il); + inpL = build_hc_post(cur, residual, attn_hc.post, il); cb(inpL, "hc_attn_out", il); residual = inpL; - cur = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, - model.layers[il].hc_ffn_base, &post, il); + hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, + model.layers[il].hc_ffn_base, il); + cur = ffn_hc.cur; cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -575,7 +580,7 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "ffn_out", il); } - inpL = build_hc_post(cur, residual, post, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, il); cb(inpL, "l_out", il); } diff --git a/src/models/models.h b/src/models/models.h index 37dbacdb91..5d9b6077ae 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1358,24 +1358,33 @@ struct llama_model_deepseek4 : public llama_model_base { // mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] ggml_tensor * build_hc_mean(ggml_tensor * x) const; - // returns the collapsed input and fills the post / comb weights - ggml_tensor * build_hc_pre( + // all outputs of the hc mix builders: + // build_hc_pre - cur is the collapsed input, post / comb feed build_hc_post + // build_hc_mixes - pre is the pre weights, post / comb feed build_hc_post + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + ggml_tensor * comb; + }; + + struct hc_mixes { + ggml_tensor * pre; + ggml_tensor * post; + ggml_tensor * comb; + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; - // returns the pre weights and fills post / comb - ggml_tensor * build_hc_mixes( + hc_mixes build_hc_mixes( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; ggml_tensor * build_hc_post( @@ -2263,12 +2272,16 @@ struct llama_model_hy_v4 : public llama_model_base { // iHC (independent Hyper-Connections): pre reduces the hc streams to one and returns the // per-stream post gates, post writes the sublayer output back into the streams, head // collapses the streams before the final norm. - ggml_tensor * build_hc_pre( + struct hc_pre { + ggml_tensor * cur; // the collapsed stream + ggml_tensor * post; // per-stream post gates, feeds build_hc_post + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, int il) const; ggml_tensor * build_hc_post( @@ -2301,12 +2314,18 @@ struct llama_model_hy_v4 : public llama_model_base { ggml_tensor * inp_pos, int il) const; - ggml_tensor * build_attention_dsa( + // the attention output and the top-k to carry to the following "shared" layers + struct attn_dsa { + ggml_tensor * cur; + ggml_tensor * top_k; + }; + + attn_dsa build_attention_dsa( const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, - ggml_tensor ** last_top_k, + ggml_tensor * last_top_k, float kq_scale, int il) const; }; @@ -2730,14 +2749,18 @@ struct llama_model_qwen4exp : public llama_model_base { llm_build_delta_net_base(params), model(model) {} // HC replaces every layer norm: residual is [n_embd, hc, n_tokens] - ggml_tensor * build_hc_mix( + struct hc_mix { + ggml_tensor * cur; // the mixed [n_embd, T] stream + ggml_tensor * inject; // [hc, T] scatter weights, null when the inject weights are null + }; + + hc_mix build_hc_mix( ggml_tensor * x, ggml_tensor * w_norm, ggml_tensor * w_down, ggml_tensor * w_up, ggml_tensor * w_inject, ggml_tensor * w_down_inject, // [w_down | w_inject] in one weight, can be null - ggml_tensor ** inject, int il); ggml_tensor * build_hc_combine( @@ -2969,14 +2992,18 @@ struct llama_model_glm5_next : public llama_model_base { // mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] ggml_tensor * build_hc_mean(ggml_tensor * x) const; - // returns the collapsed input and fills the post / comb weights - ggml_tensor * build_hc_pre( + // all outputs of the hc mix builder: cur is the collapsed input, post / comb feed build_hc_post + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + ggml_tensor * comb; + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; ggml_tensor * build_hc_post( @@ -3002,9 +3029,15 @@ struct llama_model_glm5_next : public llama_model_base { ggml_tensor * build_kpool_select(ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * kq_mask, const llama_layer & layer, const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_kpool * inp_kpool, int il); - ggml_tensor * build_dsa_layer(ggml_tensor * cur, const llama_layer & layer, - const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, - llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il); + // attention output (cur) and the selection to carry to the next shared layer (sel) + struct dsa_out { + ggml_tensor * cur; + ggml_tensor * sel; + }; + + dsa_out build_dsa_layer(ggml_tensor * cur, const llama_layer & layer, + const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, + llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il); }; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 1f3b1cdefd..f5a81f8f65 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -343,15 +343,15 @@ static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) { } // Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms. -// Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights. -ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( +// build_hc_mix returns the mixed [n_embd, T] stream and the [hc, T] scatter weights. +llama_model_qwen4exp::graph::hc_mix +llama_model_qwen4exp::graph::build_hc_mix( ggml_tensor * x, ggml_tensor * w_norm, ggml_tensor * w_down, ggml_tensor * w_up, ggml_tensor * w_inject, ggml_tensor * w_down_inject, - ggml_tensor ** inject, int il) { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc * n_embd; @@ -364,15 +364,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( cb(xn, "hc_norm", il); ggml_tensor * lo = nullptr; - if (inject && w_down_inject) { + ggml_tensor * inject = nullptr; + if (w_down_inject) { // one matmul for both: the hc_down rows, then the hc_inject rows const int64_t n_lo = w_down_inject->ne[1] - hc; ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn); // scale the combined rows before the split; both lo and inject use 1/hc lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc); - lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0); - *inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo)); - cb(*inject, "hc_inject", il); + lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0); + inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo)); + cb(inject, "hc_inject", il); } else { lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc); } @@ -405,12 +406,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( } cb(mixed, "hc_mixed", il); - if (inject && !w_down_inject) { - *inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc); - cb(*inject, "hc_inject", il); + if (!w_down_inject && w_inject) { + inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc); + cb(inject, "hc_inject", il); } - return mixed; + return { mixed, inject }; } ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( @@ -504,14 +505,14 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il); } - ggml_tensor * inject = nullptr; - ggml_tensor * cur = build_hc_mix(res_hc, + hc_mix attn_mix = build_hc_mix(res_hc, model.layers[il].hc_attn_norm, model.layers[il].hc_attn_down, model.layers[il].hc_attn_up, model.layers[il].hc_attn_inject, model.layers[il].hc_attn_down_inject, - &inject, il); + il); + ggml_tensor * cur = attn_mix.cur; ggml_build_forward_expand(gf, cur); @@ -523,28 +524,29 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { // everything below is per token, so drop the rows that produce no output - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inject = ggml_get_rows(ctx0, inject, inp_out_ids); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + attn_mix.inject = ggml_get_rows(ctx0, attn_mix.inject, inp_out_ids); res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]); res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids); res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]); } - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); - cur = build_hc_mix(res_hc, + hc_mix ffn_mix = build_hc_mix(res_hc, model.layers[il].hc_ffn_norm, model.layers[il].hc_ffn_down, model.layers[il].hc_ffn_up, model.layers[il].hc_ffn_inject, model.layers[il].hc_ffn_down_inject, - &inject, il); + il); + cur = ffn_mix.cur; cur = build_layer_ffn(cur, il); cb(cur, "ffn_out", il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il); // "l_last" is the layer output name that build_cvec and imatrix look for cb(res_hc, "l_last", il); @@ -562,9 +564,10 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa } // the final mixer is the output norm: there is no separate one - ggml_tensor * cur = build_hc_mix(res_hc, + hc_mix head_mix = build_hc_mix(res_hc, model.hc_head_norm, model.hc_head_down, model.hc_head_up, - nullptr, nullptr, nullptr, -1); + nullptr, nullptr, -1); + ggml_tensor * cur = head_mix.cur; cb(cur, "result_norm", -1); res->t_embd = cur; @@ -641,14 +644,15 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); // [n_embd, hc, n_tokens] cb(res_hc, "mtp_eh_proj", il); - ggml_tensor * inject = nullptr; - ggml_tensor * cur = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, &inject, il); + hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il); + ggml_tensor * cur = attn_mix.cur; cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); - cur = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, &inject, il); + hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il); + cur = ffn_mix.cur; cur = build_layer_ffn(cur, il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il); // the next draft step reads this residual as its h ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens); @@ -657,9 +661,10 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ cb(res->t_h_nextn, "h_nextn", il); ggml_build_forward_expand(gf, res->t_h_nextn); - cur = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]), + hc_mix head_mix = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]), layer.nextn.hc_head_norm, layer.nextn.hc_head_down, layer.nextn.hc_head_up, - nullptr, nullptr, nullptr, il); + nullptr, nullptr, il); + cur = head_mix.cur; cb(cur, "result_norm", -1); res->t_embd = cur; diff --git a/tests/llama-context/cases-kv.cpp b/tests/llama-context/cases-kv.cpp index 27e6493e45..f7189c6a0f 100644 --- a/tests/llama-context/cases-kv.cpp +++ b/tests/llama-context/cases-kv.cpp @@ -346,7 +346,7 @@ struct case_state_partial_size : ctx_test { const char * name() const override { return "ps"; } const char * description() const override { - return "the partial state is not larger than the full state and, without SWA, does not grow with the tokens"; + return "the partial state is not larger than the full state"; } case_result run(model_run & mr) const override { @@ -376,10 +376,6 @@ struct case_state_partial_size : ctx_test { CTX_REQUIRE(partial_a <= full_a && partial_b <= full_b, "the partial state is larger than the full state"); - if (llama_model_n_swa(mr.model) == 0) { - CTX_REQUIRE(partial_a == partial_b, "the partial state grows with the tokens"); - } - return case_result::pass(); } }; @@ -718,6 +714,54 @@ struct case_state_restore_fragmented : ctx_test { } }; +// +// no_alloc +// + +// - load a second instance of the model with no_alloc, so the tensor data stays on disk +// - create a context from it, which reserves the compute graphs over the dummy buffers +// - the memory use must still be reported, and nothing real may be allocated for the memory +struct case_noalloc : ctx_test { + const char * name() const override { return "noalloc"; } + + const char * description() const override { + return "with no_alloc the memory use is simulated: non-zero breakdown, null buffer bases"; + } + + case_result run(model_run & mr) const override { + const std::string & path = mr.params.model.path; + CTX_REQUIRE(!path.empty(), "the model path is not known"); + + llama_model_params mparams = llama_model_default_params(); + mparams.no_alloc = true; + mparams.load_mode = LLAMA_LOAD_MODE_NONE; + + llama_model_ptr model(llama_model_load_from_file(path.c_str(), mparams)); + CTX_REQUIRE(model, "failed to load the model with no_alloc"); + + auto ctx = init_ctx(model.get(), common_context_params_to_llama(mr.params), 0); + CTX_REQUIRE(ctx, "failed to create the context with no_alloc"); + + size_t n_ctx_mem = 0; + for (const auto & [_, data] : llama_get_memory_breakdown(ctx.get())) { + n_ctx_mem += data.context; + } + + // models without a memory module (BERT, diffusion) have a null memory + if (llama_get_memory(ctx.get()) != nullptr) { + CTX_REQUIRE(n_ctx_mem > 0, "the simulated memory breakdown is zero"); + } else { + CTX_REQUIRE(n_ctx_mem == 0, "a model without memory reported context memory"); + } + + for (ggml_backend_buffer_t buf : llama_memory_get_buffers(llama_get_memory(ctx.get()))) { + CTX_REQUIRE(ggml_backend_buffer_get_base(buf) == nullptr, "a memory buffer was really allocated"); + } + + return case_result::pass(); + } +}; + std::vector make_kv_cases() { std::vector cases; @@ -733,6 +777,7 @@ std::vector make_kv_cases() { cases.push_back(std::make_unique()); cases.push_back(std::make_unique()); cases.push_back(std::make_unique()); + cases.push_back(std::make_unique()); return cases; } diff --git a/tests/llama-context/cases-rs.cpp b/tests/llama-context/cases-rs.cpp index b549bab4f1..bc1b68f337 100644 --- a/tests/llama-context/cases-rs.cpp +++ b/tests/llama-context/cases-rs.cpp @@ -7,16 +7,14 @@ #include "common.h" -#include #include -#include #include #include -// roll back multiple sequences, then replay them in a single batch whose per-seq token count -// exceeds n_ubatch: each seq's replay spans several ubatches while its rollback restore is -// still pending. Compared against a reference context that never advanced past the rollback -// point and decodes the identical replay batch. +// roll back multiple sequences, then replay them in a single batch whose per-seq token count exceeds n_ubatch: each +// seq's replay spans several ubatches while its rollback restore is still pending. Compared against a reference context +// that never advanced past the rollback point and decodes the identical replay batch. Only the first replayed token of +// each seq is compared. struct case_multi_seq_split_replay : ctx_test { test_group group() const override { return TEST_GROUP_RS; } @@ -94,12 +92,16 @@ struct case_multi_seq_split_replay : ctx_test { ok = decode_replay(ctx_roll.get()) && decode_replay(ctx_ref.get()); CTX_REQUIRE(ok, "the multi-seq replay decode failed"); - // both contexts decode identical batches, so the logits should match; + // the first replayed token of each seq reads the state right after the rollback, so its + // logits should match; the later tokens are not compared because small state + // differences at the start grow with each replayed token // random dummy models can still drift up to ~1.7e-5, so the bound is 1e-4 constexpr float nmse_eps = 1e-4f; logits_diff diff; - for (uint32_t i = 0; i < n_seqs*n_replay; ++i) { + for (uint32_t s = 0; s < n_seqs; ++s) { + const uint32_t i = s*n_replay; // first replayed token of this seq + const float * l_roll = llama_get_logits_ith(ctx_roll.get(), i); const float * l_ref = llama_get_logits_ith(ctx_ref.get(), i); if (l_roll == nullptr || l_ref == nullptr) { @@ -116,6 +118,29 @@ struct case_multi_seq_split_replay : ctx_test { TLOG_INF("%s: multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff.diff_max, diff.nmse()); + // the replay above can leave the 2 contexts in slightly different states, and the tail + // below assumes they start from the same state - so copy the roll state into ref + for (uint32_t s = 0; s < n_seqs && ok; ++s) { + std::vector state; + + ok = ok && get_seq_state(ctx_roll.get(), (llama_seq_id) s, 0, state); + ok = ok && set_seq_state(ctx_ref.get(), (llama_seq_id) s, 0, state); + + // do a second round of state save/load to negate any SWA-positioning effects + if (ok) { + std::vector restored; + ok = ok && get_seq_state(ctx_ref.get(), (llama_seq_id) s, 0, restored); + + // what was loaded into ref must come back out bit-identical + CTX_REQUIRE(ok && state == restored, + "the seq " + std::to_string(s) + " state did not round-trip bit-exact (sizes " + + std::to_string(state.size()) + "/" + std::to_string(restored.size()) + ")"); + + ok = ok && set_seq_state(ctx_roll.get(), (llama_seq_id) s, 0, restored); + } + } + CTX_REQUIRE(ok, "the multi-seq state copy failed"); + // seq-1-only decodes must be independent of seq 0's content: diverge seq 0 // in ctx_ref only, then compare identical seq-1-only continuations constexpr uint32_t n_tail = 4; @@ -148,7 +173,7 @@ struct case_multi_seq_split_replay : ctx_test { diff_tail.add(l_roll, l_ref, n_vocab); } - CTX_REQUIRE(ok && diff_tail.nmse() <= nmse_eps, + CTX_REQUIRE(ok && diff_tail.nmse() <= 0.0f, "a seq-1-only decode leaked the state of seq 0 (nmse " + std::to_string(diff_tail.nmse()) + ")"); TLOG_INF("%s: seq-1-only decode is independent of seq 0 (max diff %g, nmse %g)\n", diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9a4ddaa29a..0ccde35ebe 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3679,23 +3679,37 @@ struct test_scale : public test_case { float scale; float bias; bool inplace; + int v; // view (1 : non-contiguous a) std::string vars() override { - return VARS_TO_STR5(type, ne, scale, bias, inplace); + return VARS_TO_STR6(type, ne, scale, bias, inplace, v); } test_scale(ggml_type type = GGML_TYPE_F32, std::array ne = {10, 10, 10, 10}, float scale = 2.0f, float bias = 0.0f, - bool inplace = false) - : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {} + bool inplace = false, + int v = 0) + : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace), v(v) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + auto ne_a = ne; + if (v & 1) { + ne_a[0] *= 3; + ne_a[1] *= 2; + ne_a[2] *= 5; + ne_a[3] *= 4; + } + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); ggml_set_param(a); ggml_set_name(a, "a"); + if (v & 1) { + a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } + ggml_tensor * out; if (inplace) { out = ggml_scale_bias_inplace(ctx, a, scale, bias); @@ -7126,7 +7140,7 @@ struct test_concat : public test_case { const std::array ne_a; const int64_t ne_b_d; const int dim; - const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim)) + const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim), 1 << 4: transposed b) std::string vars() override { return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v); @@ -7175,6 +7189,13 @@ struct test_concat : public test_case { b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0); ggml_set_name(b, "view_of_b"); + } else if (v & 16) { + auto ne = ne_b; std::swap(ne[0], ne[1]); + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + b = ggml_transpose(ctx, b); + ggml_set_name(b, "transpose_of_b"); } else { b = ggml_new_tensor(ctx, type, 4, ne_b.data()); ggml_set_name(b, "b"); @@ -10854,6 +10875,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f)); test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f)); + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace + test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f)); test_cases.emplace_back(new test_silu_back()); @@ -10893,9 +10918,20 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_l2_norm (GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-12f, false, false)); test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-6f, false, false)); + for (uint32_t n : {1, 127, 129, 511, 513, 2816, 4096, 16385}) { + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {n, 3, 2, 1})); + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 3, 2, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 3, 2, 1}, 1e-6f, false, false, false, false, true)); + } + for (ggml_type set_rows_type : { GGML_TYPE_F32, GGML_TYPE_F16 }) { test_cases.emplace_back(new test_rms_norm_mul_rope({ 256, 1, 1, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); test_cases.emplace_back(new test_rms_norm_mul_rope({ 128, 4, 3, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type)); + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + for (int64_t n : {64, 256, 512, 1024}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({n, 4, 3, 1}, 1e-6f, false, true, true, mode, true, true, set_rows_type)); + } + } } for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) { @@ -11792,7 +11828,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global - for (int v : { 0, 1, 2, 3 }) { + for (int v : { 0, 1, 2, 3, 16 }) { for (int dim : { 0, 1, 2, 3, }) { test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v)); @@ -12520,6 +12556,9 @@ static std::vector> make_test_cases_eval() { static std::vector> make_test_cases_perf() { std::vector> test_cases; + // delta-net conv input: conv state followed by the transposed qkv of the ubatch + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16)); + // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { @@ -12549,6 +12588,15 @@ static std::vector> make_test_cases_perf() { } } + // RMS_NORM and fused variants at decode and prefill shapes + for (int64_t n_tokens : {1, 512}) { + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {4096, n_tokens, 1, 1})); + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {128, 32*n_tokens, 1, 1})); + test_cases.emplace_back(new test_rms_norm_mul_rope({4096, n_tokens, 1, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, n_tokens, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NEOX)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 8, n_tokens, 1}, 1e-6f, false, true, true, GGML_ROPE_TYPE_NEOX)); + } + // POOL_1D max over MoE down-projection amax shapes: // amax is [1, n_expert, n_tokens] after reducing the whole intermediate dim. for (int64_t n_ff : { 2048, 4096, 8192 }) { diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index eb7b30936e..705fd65fa9 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -458,6 +458,2255 @@ static void test_shared_removal() { fs::remove_all(cache_dir / "models--test--alone"); } +// +// real repo listings captured from the HF API: GGUF repos whose file names follow +// the GGUF naming convention, one listing per layout pattern, each pinned to the +// plans and the cache listing the resolver gives today, so any change in +// resolution shows up here +// + +// a file name, or with `parts` a shard prefix standing for the verbatim names +// -00001-of-0000n.gguf .. -0000n-of-0000n.gguf +struct real_file { + const char * name; + int parts = 0; +}; + +struct real_plan { + const char * tag; // the request is :, or when empty + const char * primary; + int parts; // number of model files, all splits of the primary + const char * mmproj; + const char * mtp; + const char * dflash; + const char * eagle3; + const char * dspark; +}; + +struct real_repo { + const char * id; + std::vector files; + std::vector plans; + std::vector listing; // the tags the cache listing reports +}; + +static const real_repo real_repos[] = { + {"0bserverx/Muse-Glimmer-30B-Heretic-Uncensored-GGUF", + {{"Muse-Glimmer-30B-Heretic-BF16.gguf"}, + {"Muse-Glimmer-30B-Heretic-IQ4_XS.gguf"}, + {"Muse-Glimmer-30B-Heretic-Q4_K_M.gguf"}, + {"Muse-Glimmer-30B-Heretic-Q8_0.gguf"}, + {"Muse-Glimmer-30B-Heretic-TQ2_0.gguf"}}, + { + {"", "Muse-Glimmer-30B-Heretic-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Muse-Glimmer-30B-Heretic-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Muse-Glimmer-30B-Heretic-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "Muse-Glimmer-30B-Heretic-BF16.gguf", 1, "", "", "", "", ""}, + {"TQ2_0", "Muse-Glimmer-30B-Heretic-TQ2_0.gguf", 1, "", "", "", "", ""}}, + {"TQ2_0", "Q8_0", "Q4_K_M", "IQ4_XS", "BF16"}}, + + {"0bserverx/RVN-Qwen3.8-Flash-Next-Abliterated-Uncensored-GGUF", + {{"RVN-Qwen3.8-Flash-Next-IQ3_M", 8}, + {"RVN-Qwen3.8-Flash-Next-Q4_0", 8}, + {"RVN-Qwen3.8-Flash-Next-Q4_K_M", 8}, + {"RVN-Qwen3.8-Flash-Next-Q6_K", 8}}, + { + {"", "RVN-Qwen3.8-Flash-Next-Q4_K_M-00001-of-00008.gguf", 8, "", "", "", "", ""}, + {"Q4_K_M", "RVN-Qwen3.8-Flash-Next-Q4_K_M-00001-of-00008.gguf", 8, "", "", "", "", ""}, + {"IQ3_M", "RVN-Qwen3.8-Flash-Next-IQ3_M-00001-of-00008.gguf", 8, "", "", "", "", ""}, + {"Q6_K", "RVN-Qwen3.8-Flash-Next-Q6_K-00001-of-00008.gguf", 8, "", "", "", "", ""}}, + {"Q6_K", "Q4_K_M", "Q4_0", "IQ3_M"}}, + + {"0xSojalSec/Qwen3.8-Flash-Next-Uncensored-GSQ-RCO-12-GB-GPU", + {{"IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS", 2}, + {"IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ3_XXS", 2}, + {"Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-Q2_0", 2}, + {"mmproj-Qwen3.8-Flash-Next-BF16.gguf"}, + {"mtp-Qwen3.8-Flash-Next-Q8_0.gguf"}, + {"mtp-q2_0.gguf"}}, + { + {"", "IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"Q8_0", "", 0, "", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"IQ2_XS", "IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"Q2_0", "Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-abliterated-Q2_0-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"BF16", "", 0, "", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}}, + {"Q2_0", "IQ3_XXS", "IQ2_XS"}}, + + {"agentionai/Qwen3.8-Flash-Next-AP-GGUF", + {{"AP-IQ2_S/Qwen3.8-Flash-Next-AP-IQ2_S.gguf"}, + {"AP-Q4_K_M/Qwen3.8-Flash-Next-AP-Q4_K_M.gguf"}, + {"AP-Q5_K_XL/Qwen3.8-Flash-Next-AP-Q5_K_XL.gguf"}, + {"mmproj-F16.gguf"}}, + { + {"", "AP-Q4_K_M/Qwen3.8-Flash-Next-AP-Q4_K_M.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "AP-Q4_K_M/Qwen3.8-Flash-Next-AP-Q4_K_M.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"IQ2_S", "AP-IQ2_S/Qwen3.8-Flash-Next-AP-IQ2_S.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"Q5_K_XL", "AP-Q5_K_XL/Qwen3.8-Flash-Next-AP-Q5_K_XL.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q5_K_XL", "Q4_K_M", "IQ2_S"}}, + + {"agentionai/Signal-3.8-27B-GGUF", + {{"AP-IQ3_S/Signal-3.8-27B-AP-IQ3_S.gguf"}, + {"AP-Q4_K_M/Signal-3.8-27B-AP-Q4_K_M.gguf"}, + {"Q8_0/Signal-3.8-27B-Q8_0.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "AP-Q4_K_M/Signal-3.8-27B-AP-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "AP-Q4_K_M/Signal-3.8-27B-AP-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Signal-3.8-27B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"IQ3_S", "AP-IQ3_S/Signal-3.8-27B-AP-IQ3_S.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "IQ3_S"}}, + + {"agentionai/Signal-3.8-Flash-Next-GGUF", + {{"AP-IQ4_XS/Signal-3.8-Flash-Next-AP-IQ4_XS.gguf"}, + {"AP-Q4_K_XL/Signal-3.8-Flash-Next-AP-Q4_K_XL.gguf"}, + {"Q8_0/Signal-3.8-Flash-Next-Q8_0.gguf"}, + {"mmproj-F16.gguf"}}, + { + {"", "Q8_0/Signal-3.8-Flash-Next-Q8_0.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Signal-3.8-Flash-Next-Q8_0.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"IQ4_XS", "AP-IQ4_XS/Signal-3.8-Flash-Next-AP-IQ4_XS.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_XL", "IQ4_XS"}}, + + {"ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF", + {{"RCO/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ2_S.gguf"}, + {"Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q3_K_XL.gguf"}, + {"Swift-1.5-Qwen3.8-27B-Uncensored-MTP-BF16.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "RCO/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ2_S.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"IQ2_S", "RCO/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ2_S.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "Swift-1.5-Qwen3.8-27B-Uncensored-MTP-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"BF16", "Q3_K_XL", "IQ2_S"}}, + + {"ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF", + {{"Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-IQ3_S.gguf"}, + {"Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q4_K_XL.gguf"}, + {"Swift-Qwen3.8-27B-Uncensored-MTP-BF16.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-IQ3_S.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-IQ3_S", "Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-IQ3_S.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "Swift-Qwen3.8-27B-Uncensored-MTP-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"BF16", "Q4_K_XL", "IQ3_S"}}, + + {"AliceThirty/GLM-5.3-Flash-UNCENSORED-GGUF", + {{"GLM-5.3-Flash-UNCENSORED-FP8-mmproj-bf16.gguf"}, + {"UD-Q3_K_XL/GLM-5.3-Flash-UNCENSORED-FP8-UD-Q3_K_XL", 4}, + {"UD-Q5_K_XL/GLM-5.3-Flash-UNCENSORED-FP8-UD-Q5_K_XL", 5}}, + { + {"", "UD-Q3_K_XL/GLM-5.3-Flash-UNCENSORED-FP8-UD-Q3_K_XL-00001-of-00004.gguf", 4, "GLM-5.3-Flash-UNCENSORED-FP8-mmproj-bf16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}, + {"UD-Q5_K_XL", "UD-Q5_K_XL/GLM-5.3-Flash-UNCENSORED-FP8-UD-Q5_K_XL-00001-of-00005.gguf", 5, "GLM-5.3-Flash-UNCENSORED-FP8-mmproj-bf16.gguf", "", "", "", ""}}, + {"Q5_K_XL", "Q3_K_XL"}}, + + {"antirez/glm-5.3-gguf", + {{"GLM-5.3-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf"}}, + { + {"", "GLM-5.3-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf", 1, "", "", "", "", ""}, + {"UD-IQ2_XXS_ROUTEDIQ2XXS_BLK78Q2K", "GLM-5.3-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf", 1, "", "", "", "", ""}}, + {"IQ2_XXS_ROUTEDIQ2XXS_BLK78Q2K"}}, + + {"AtomicChat/Qwen-Image-2.1-Turbo-GGUF", + {{"Qwen-Image-2.1-Turbo-AD-Q2_K.gguf"}, + {"Qwen-Image-2.1-Turbo-AD-Q5_K.gguf"}, + {"Qwen-Image-2.1-Turbo-Q8_0.gguf"}}, + { + {"", "Qwen-Image-2.1-Turbo-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Qwen-Image-2.1-Turbo-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q2_K", "Qwen-Image-2.1-Turbo-AD-Q2_K.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K", "Q2_K"}}, + + {"AtomicChat/Qwen-Image-2.1-Turbo-Uncensored-GGUF", + {{"Qwen-Image-2.1-Turbo-Uncensored-AD-Q2_K.gguf"}, + {"Qwen-Image-2.1-Turbo-Uncensored-AD-Q5_K.gguf"}, + {"Qwen-Image-2.1-Turbo-Uncensored-Q8_0.gguf"}, + {"mmproj-Qwen-Image-2.1-Turbo-Uncensored-BF16.gguf"}, + {"mmproj-Qwen-Image-2.1-Turbo-Uncensored-F16.gguf"}}, + { + {"", "Qwen-Image-2.1-Turbo-Uncensored-Q8_0.gguf", 1, "mmproj-Qwen-Image-2.1-Turbo-Uncensored-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen-Image-2.1-Turbo-Uncensored-Q8_0.gguf", 1, "mmproj-Qwen-Image-2.1-Turbo-Uncensored-BF16.gguf", "", "", "", ""}, + {"Q2_K", "Qwen-Image-2.1-Turbo-Uncensored-AD-Q2_K.gguf", 1, "mmproj-Qwen-Image-2.1-Turbo-Uncensored-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q5_K", "Q2_K"}}, + + {"audio-cpp/Audio-Flamingo-GGUF", + {{"audio-flamingo-3-bf16.gguf"}, + {"audio-flamingo-3-q4_k.gguf"}, + {"audio-flamingo-3-q8_0.gguf"}, + {"audio-flamingo-next-bf16.gguf"}, + {"audio-flamingo-next-q4_k.gguf"}, + {"audio-flamingo-next-q8_0.gguf"}}, + { + {"", "audio-flamingo-3-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "audio-flamingo-3-q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "audio-flamingo-3-bf16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K", "BF16"}}, + + {"bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF", + {{"DeepSeek-Coder-V2-Lite-Instruct-IQ2_M.gguf"}, + {"DeepSeek-Coder-V2-Lite-Instruct-Q4_K_L.gguf"}, + {"DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf"}, + {"DeepSeek-Coder-V2-Lite-Instruct-Q8_0.gguf"}, + {"DeepSeek-Coder-V2-Lite-Instruct-f32.gguf/DeepSeek-Coder-V2-Lite-Instruct-f32", 2}}, + { + {"", "DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "DeepSeek-Coder-V2-Lite-Instruct-Q8_0.gguf", 1, "", "", "", "", ""}, + {"IQ2_M", "DeepSeek-Coder-V2-Lite-Instruct-IQ2_M.gguf", 1, "", "", "", "", ""}, + {"F32", "DeepSeek-Coder-V2-Lite-Instruct-f32.gguf/DeepSeek-Coder-V2-Lite-Instruct-f32-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"F32", "Q8_0", "Q4_K_M", "Q4_K_L", "IQ2_M"}}, + + {"bartowski/DeepSeek-R1-Distill-Qwen-14B-GGUF", + {{"DeepSeek-R1-Distill-Qwen-14B-IQ2_M.gguf"}, + {"DeepSeek-R1-Distill-Qwen-14B-Q4_0.gguf"}, + {"DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf"}, + {"DeepSeek-R1-Distill-Qwen-14B-Q8_0.gguf"}, + {"DeepSeek-R1-Distill-Qwen-14B-f32/DeepSeek-R1-Distill-Qwen-14B-f32", 2}}, + { + {"", "DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "DeepSeek-R1-Distill-Qwen-14B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"IQ2_M", "DeepSeek-R1-Distill-Qwen-14B-IQ2_M.gguf", 1, "", "", "", "", ""}, + {"F32", "DeepSeek-R1-Distill-Qwen-14B-f32/DeepSeek-R1-Distill-Qwen-14B-f32-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"F32", "Q8_0", "Q4_K_M", "Q4_0", "IQ2_M"}}, + + {"bartowski/Qwen2.5-72B-Instruct-GGUF", + {{"Qwen2.5-72B-Instruct-IQ1_M.gguf"}, + {"Qwen2.5-72B-Instruct-Q3_K_L.gguf"}, + {"Qwen2.5-72B-Instruct-Q4_K_M.gguf"}, + {"Qwen2.5-72B-Instruct-Q8_0/Qwen2.5-72B-Instruct-Q8_0", 2}}, + { + {"", "Qwen2.5-72B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen2.5-72B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Qwen2.5-72B-Instruct-Q8_0/Qwen2.5-72B-Instruct-Q8_0-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"IQ1_M", "Qwen2.5-72B-Instruct-IQ1_M.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_L", "IQ1_M"}}, + + {"batiai/gemma-4-E2B-it-GGUF", + {{"google-gemma-4-E2B-it-Q4_K_M.gguf"}, + {"google-gemma-4-E2B-it-Q6_K.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "google-gemma-4-E2B-it-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "google-gemma-4-E2B-it-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q6_K", "google-gemma-4-E2B-it-Q6_K.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q6_K", "Q4_K_M"}}, + + {"batiai/Qwen3.6-27B-GGUF", + {{"Qwen-Qwen3.6-27B-IQ3_XXS.gguf"}, + {"Qwen-Qwen3.6-27B-Q3_K_M.gguf"}, + {"Qwen-Qwen3.6-27B-Q4_K_M.gguf"}, + {"Qwen-Qwen3.6-27B-Q6_K.gguf"}, + {"mmproj-Qwen-Qwen3.6-27B-BF16.gguf"}, + {"mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf"}}, + { + {"", "Qwen-Qwen3.6-27B-Q4_K_M.gguf", 1, "mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen-Qwen3.6-27B-Q4_K_M.gguf", 1, "mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf", "", "", "", ""}, + {"IQ3_XXS", "Qwen-Qwen3.6-27B-IQ3_XXS.gguf", 1, "mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf", "", "", "", ""}, + {"Q6_K", "Qwen-Qwen3.6-27B-Q6_K.gguf", 1, "mmproj-Qwen-Qwen3.6-27B-Q6_K.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q6_K", "Q4_K_M", "Q3_K_M", "IQ3_XXS"}}, + + {"BattleGhost/Qwen3.8-Flash-Next-UNCENSORED-IQ2_XXS-GGUF", + {{"Qwen3.8-Flash-Next-UNCENSORED-IQ2_XXS.gguf"}}, + { + {"", "Qwen3.8-Flash-Next-UNCENSORED-IQ2_XXS.gguf", 1, "", "", "", "", ""}, + {"IQ2_XXS", "Qwen3.8-Flash-Next-UNCENSORED-IQ2_XXS.gguf", 1, "", "", "", "", ""}}, + {"IQ2_XXS"}}, + + {"befox/WAN2.2-14B-Rapid-AllInOne-GGUF", + {{"Mega-v1/wan2.2-rapid-mega-aio-nsfw-v1-Q2_K.gguf"}, + {"Mega-v1/wan2.2-rapid-mega-aio-nsfw-v1-Q8_0.gguf"}, + {"Mega-v1/wan2.2-rapid-mega-aio-v1-Q2_K.gguf"}, + {"Mega-v1/wan2.2-rapid-mega-aio-v1-Q8_0.gguf"}, + {"Mega-v10/wan2.2-rapid-mega-aio-nsfw-v10-Q8_0.gguf"}, + {"Mega-v10/wan2.2-rapid-mega-aio-v10-Q8_0.gguf"}, + {"Mega-v12/wan2.2-rapid-mega-aio-nsfw-v12.1-Q8_0.gguf"}, + {"Mega-v12/wan2.2-rapid-mega-aio-v12-Q8_0.gguf"}, + {"Mega-v2/wan2.2-rapid-mega-aio-nsfw-v2-Q8_0.gguf"}, + {"Mega-v2/wan2.2-rapid-mega-aio-v2-Q8_0.gguf"}, + {"Mega-v3/wan2.2-rapid-mega-aio-v3-Q5_K.gguf"}, + {"Mega-v3/wan2.2-rapid-mega-aio-v3-Q8_0.gguf"}, + {"Mega-v3/wan2.2-rapid-mega-nsfw-aio-v3.1-Q5_K.gguf"}, + {"Mega-v3/wan2.2-rapid-mega-nsfw-aio-v3.1-Q8_0.gguf"}, + {"Mega-v4/wan2.2-rapid-mega-aio-v4-Q8_0.gguf"}, + {"Mega-v5/wan2.2-rapid-mega-aio-nsfw-v5-Q8_0.gguf"}, + {"Mega-v5/wan2.2-rapid-mega-aio-v5-Q8_0.gguf"}, + {"Mega-v7/wan2.2-rapid-mega-aio-v7-Q8_0.gguf"}, + {"v10/wan2.2-i2v-rapid-aio-v10-Q8_0.gguf"}, + {"v10/wan2.2-i2v-rapid-aio-v10-nsfw-Q8_0.gguf"}, + {"v10/wan2.2-t2v-rapid-aio-v10-Q8_0.gguf"}, + {"v10/wan2.2-t2v-rapid-aio-v10-nsfw-Q8_0.gguf"}}, + { + {"", "Mega-v1/wan2.2-rapid-mega-aio-nsfw-v1-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Mega-v1/wan2.2-rapid-mega-aio-nsfw-v1-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q2_K", "Mega-v1/wan2.2-rapid-mega-aio-nsfw-v1-Q2_K.gguf", 1, "", "", "", "", ""}, + {"Q5_K", "Mega-v3/wan2.2-rapid-mega-aio-v3-Q5_K.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K", "Q2_K"}}, + + {"Blackfrost-AI/Qwen3.8-27B-ABLITERATED-GGUF", + {{"Qwen3.8-27B-ABLITERATED-Q2_K.gguf"}, + {"Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf"}, + {"Qwen3.8-27B-ABLITERATED-Q4_K_S.gguf"}, + {"Qwen3.8-27B-ABLITERATED-Q8_0.gguf"}, + {"dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf"}, + {"mmproj-Qwen3.8-27B-ABLITERATED-F16.gguf"}, + {"mmproj-Qwen3.8-27B-ABLITERATED-Q8_0.gguf"}}, + { + {"", "Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-ABLITERATED-Q8_0.gguf", "", "dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-ABLITERATED-Q8_0.gguf", "", "dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf", "", ""}, + {"Q8_0", "Qwen3.8-27B-ABLITERATED-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-ABLITERATED-Q8_0.gguf", "", "dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf", "", ""}, + {"Q2_K", "Qwen3.8-27B-ABLITERATED-Q2_K.gguf", 1, "mmproj-Qwen3.8-27B-ABLITERATED-Q8_0.gguf", "", "dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf", "", ""}, + {"BF16", "", 0, "", "", "dflash-Qwen3.8-27B-ABLITERATED-BF16.gguf", "", ""}}, + {"Q8_0", "Q4_K_S", "Q4_K_M", "Q2_K"}}, + + {"bloomer010/Ling-3.0-flash-GGUF", + {{"Ling-3.0-flash-BF16.gguf"}, + {"Ling-3.0-flash-Q4_K_M.gguf"}, + {"Ling-3.0-flash-Q4_K_S.gguf"}, + {"Ling-3.0-flash-Q8_0.gguf"}, + {"Ling-3.0-flash-UD-Q8_K_XL.gguf"}, + {"Ling-3.0-flash-dspark-BF16.gguf"}, + {"Ling-3.0-flash-dspark-Q4_K_M.gguf"}}, + { + {"", "Ling-3.0-flash-Q4_K_M.gguf", 1, "", "", "", "", "Ling-3.0-flash-dspark-Q4_K_M.gguf"}, + {"Q4_K_M", "Ling-3.0-flash-Q4_K_M.gguf", 1, "", "", "", "", "Ling-3.0-flash-dspark-Q4_K_M.gguf"}, + {"Q8_0", "Ling-3.0-flash-Q8_0.gguf", 1, "", "", "", "", "Ling-3.0-flash-dspark-Q4_K_M.gguf"}, + {"BF16", "Ling-3.0-flash-BF16.gguf", 1, "", "", "", "", "Ling-3.0-flash-dspark-BF16.gguf"}, + {"UD-Q8_K_XL", "Ling-3.0-flash-UD-Q8_K_XL.gguf", 1, "", "", "", "", "Ling-3.0-flash-dspark-Q4_K_M.gguf"}}, + {"Q8_K_XL", "Q8_0", "Q4_K_S", "Q4_K_M", "BF16"}}, + + {"BnhHo/Qwen3.8-27B-Uncensored-GGUF", + {{"Qwen3.8-27B-Uncensored-IQ2_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q5_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q8_0.gguf"}, + {"Qwen3.8-27B-Uncensored-draft-Q4_0.gguf"}, + {"Qwen3.8-27B-Uncensored-draft-Q8_0.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-IQ2_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q5_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-Uncensored-F16.gguf"}}, + { + {"", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-Uncensored-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"IQ2_M", "Qwen3.8-27B-Uncensored-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q4_0", "Qwen3.8-27B-Uncensored-draft-Q4_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ2_M", "Q4_0"}}, + + {"bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF", + {{"ThinkingCap-Qwen3.8-27B-IQ4_XS.gguf"}, + {"ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf"}, + {"ThinkingCap-Qwen3.8-27B-Q6_K.gguf"}, + {"ThinkingCap-Qwen3.8-27B-Q8_0.gguf"}, + {"ThinkingCap-Qwen3.8-27B-f16.gguf"}, + {"mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf"}}, + { + {"", "ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf", 1, "mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf", 1, "mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf", "", "", "", ""}, + {"Q8_0", "ThinkingCap-Qwen3.8-27B-Q8_0.gguf", 1, "mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf", "", "", "", ""}, + {"IQ4_XS", "ThinkingCap-Qwen3.8-27B-IQ4_XS.gguf", 1, "mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf", "", "", "", ""}, + {"F16", "ThinkingCap-Qwen3.8-27B-f16.gguf", 1, "mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf", "", "", "", ""}}, + {"F16", "Q8_0", "Q6_K", "Q4_K_M", "IQ4_XS"}}, + + {"bytkim/Qwen3.8-27B-pi-GGUF", + {{"Qwen3.8-27B-pi-BF16.gguf"}, + {"Qwen3.8-27B-pi-Q3_K_M.gguf"}, + {"Qwen3.8-27B-pi-Q4_K_M.gguf"}, + {"Qwen3.8-27B-pi-Q8_0.gguf"}, + {"dflash2/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"}, + {"dflash2/Qwen3.8-27B-DFlash2-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-pi-BF16.gguf"}, + {"mtp/mtp-Qwen3.8-27B-pi-BF16.gguf"}, + {"mtp/mtp-Qwen3.8-27B-pi-Q4_0.gguf"}, + {"mtp/mtp-Qwen3.8-27B-pi-Q8_0.gguf"}}, + { + {"", "Qwen3.8-27B-pi-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-pi-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-pi-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-pi-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-pi-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-pi-BF16.gguf", "", "", "", ""}, + {"BF16", "Qwen3.8-27B-pi-BF16.gguf", 1, "mmproj-Qwen3.8-27B-pi-BF16.gguf", "", "", "", ""}, + {"Q4_0", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_M", "BF16"}}, + + {"CaptainArni/audio.cpp-gguf", + {{"ACE-Step1.5-GGUF/xl-sft-q8dit/ace-step-1.5-xl-sft-q8dit.gguf"}, + {"ACE-Step1.5-GGUF/xl-turbo-q8dit/ace-step-1.5-xl-turbo-q8dit.gguf"}, + {"ACE-Step1.5-GGUF/xl-turbo/ace-step-1.5-xl-turbo-bf16.gguf"}}, + { + {"", "ACE-Step1.5-GGUF/xl-sft-q8dit/ace-step-1.5-xl-sft-q8dit.gguf", 1, "", "", "", "", ""}, + {"Q8DIT", "ACE-Step1.5-GGUF/xl-sft-q8dit/ace-step-1.5-xl-sft-q8dit.gguf", 1, "", "", "", "", ""}, + {"BF16", "ACE-Step1.5-GGUF/xl-turbo/ace-step-1.5-xl-turbo-bf16.gguf", 1, "", "", "", "", ""}}, + {"BF16", "Q8DIT"}}, + + {"cesarsal1nas/Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-Q4_K_M-GGUF", + {{"Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-Q4_K_M.gguf"}, + {"Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-mmproj-F16.gguf"}}, + { + {"", "Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-Q4_K_M.gguf", 1, "Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-mmproj-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-Q4_K_M.gguf", 1, "Huihui-Qwen3.5-35B-A3B-Claude-4.6-Opus-abliterated-mmproj-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"chimingw/Qwen3.8-27B-Uncensored-OrcaRouter-GGUF", + {{"AUX/mmproj-Qwen3.8-27B-Uncensored-OrcaRouter-BF16.gguf"}, + {"AUX/mmproj-Qwen3.8-27B-Uncensored-OrcaRouter-F16.gguf"}, + {"BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16", 2}, + {"MTP/Qwen3.8-27B-Uncensored-OrcaRouter-MTP-Q8_0.gguf"}, + {"Qwen3.8-27B-Uncensored-OrcaRouter-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-OrcaRouter-Q5_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-OrcaRouter-Q8_0.gguf"}}, + { + {"", "Qwen3.8-27B-Uncensored-OrcaRouter-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Uncensored-OrcaRouter-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "MTP/Qwen3.8-27B-Uncensored-OrcaRouter-MTP-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"Q5_K_M", "Qwen3.8-27B-Uncensored-OrcaRouter-Q5_K_M.gguf", 1, "", "", "", "", ""}, + {"F16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, + + {"ChrisColeTech/qwen-image-edit-uncensored-GGUF", + {{"split/diffusion_models/qwen-image-edit-2511-uncensored-Q4_K_M.gguf"}, + {"split/diffusion_models/qwen-image-edit-2511-uncensored-Q8_0.gguf"}, + {"split/text_encoders/Qwen2.5-VL-7B-Instruct-q4_0.gguf"}}, + { + {"", "split/diffusion_models/qwen-image-edit-2511-uncensored-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "split/diffusion_models/qwen-image-edit-2511-uncensored-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "split/diffusion_models/qwen-image-edit-2511-uncensored-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q4_0", "split/text_encoders/Qwen2.5-VL-7B-Instruct-q4_0.gguf", 1, "", "", "", "", ""}}, + {"Q4_0", "Q8_0", "Q4_K_M"}}, + + {"city96/FLUX.1-dev-gguf", + {{"flux1-dev-F16.gguf"}, + {"flux1-dev-Q4_K_S.gguf"}, + {"flux1-dev-Q8_0.gguf"}}, + { + {"", "flux1-dev-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "flux1-dev-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "flux1-dev-F16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_S", "F16"}}, + + {"culturerevolt/gemma-4-12b-heretic-abliterated-GGUF", + {{"gemma-4-12b-heretic-IQ3_XS.gguf"}, + {"gemma-4-12b-heretic-Q4_K_M.gguf"}, + {"gemma-4-12b-heretic-Q5_K_M.gguf"}, + {"gemma-4-12b-heretic-abliterated-Q8_0.gguf"}, + {"gemma-4-12b-heretic-mmproj-f16.gguf"}}, + { + {"", "gemma-4-12b-heretic-Q4_K_M.gguf", 1, "gemma-4-12b-heretic-mmproj-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "gemma-4-12b-heretic-Q4_K_M.gguf", 1, "gemma-4-12b-heretic-mmproj-f16.gguf", "", "", "", ""}, + {"Q8_0", "gemma-4-12b-heretic-abliterated-Q8_0.gguf", 1, "gemma-4-12b-heretic-mmproj-f16.gguf", "", "", "", ""}, + {"IQ3_XS", "gemma-4-12b-heretic-IQ3_XS.gguf", 1, "gemma-4-12b-heretic-mmproj-f16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ3_XS"}}, + + {"CypherTroopers/CPH-Community-7B", + {{"cph-community-7b-q4_k_m.gguf"}}, + { + {"", "cph-community-7b-q4_k_m.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "cph-community-7b-q4_k_m.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"DamRsn/muscriptor-gguf", + {{"v1/muscriptor-large-f16.gguf"}, + {"v1/muscriptor-medium-f16.gguf"}, + {"v1/muscriptor-small-f16.gguf"}}, + { + {"", "v1/muscriptor-large-f16.gguf", 1, "", "", "", "", ""}, + {"F16", "v1/muscriptor-large-f16.gguf", 1, "", "", "", "", ""}}, + {"F16"}}, + + {"DavidAU/LFM2.5-2.6B-Qwen3.8-Turbo-Brilliance-Power-X12-NEO-MAX-GGUF", + {{"LFM2.5-2.6B-Q3.8-TBrilliance-BF16.gguf"}, + {"LFM2.5-2.6B-Q3.8-TBrilliance-NEO-MAX-Q6_K.gguf"}, + {"LFM2.5-2.6B-Q3.8-TBrilliance-NEO-MAX-Q8_0.gguf"}, + {"LFM2.5-2.6B-Q3.8-TBrilliance-NEO-Q6_K.gguf"}, + {"LFM2.5-2.6B-Q3.8-TBrilliance-NEO-Q8_0.gguf"}}, + { + {"", "LFM2.5-2.6B-Q3.8-TBrilliance-NEO-MAX-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "LFM2.5-2.6B-Q3.8-TBrilliance-NEO-MAX-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "LFM2.5-2.6B-Q3.8-TBrilliance-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q6_K", "BF16"}}, + + {"DavidAU/OpenAi-GPT-oss-20b-HERETIC-uncensored-NEO-Imatrix-gguf", + {{"OpenAI-20B-NEO-CODE-DI-Uncensored-Q5_1.gguf"}, + {"OpenAI-20B-NEO-CODE-DI-Uncensored-Q8_0.gguf"}, + {"OpenAI-20B-NEO-CODEPlus-Uncensored-IQ4_NL.gguf"}, + {"OpenAI-20B-NEO-CODEPlus-Uncensored-Q5_1.gguf"}, + {"OpenAI-20B-NEO-CODEPlus16-Uncensored-IQ4_NL.gguf"}, + {"OpenAI-20B-NEO-HRR-CODE-5-TRI-Uncensored-Q8_0.gguf"}, + {"OpenAI-20B-NEO-HRR-CODE-TRI-Uncensored-IQ4_NL.gguf"}, + {"OpenAI-20B-NEO-HRR-CODE-TRI-Uncensored-Q5_1.gguf"}, + {"OpenAI-20B-NEO-HRR-CODE-TRI-Uncensored-Q8_0.gguf"}, + {"OpenAI-20B-NEO-HRR-DI-Uncensored-Q5_1.gguf"}, + {"OpenAI-20B-NEO-HRR-DI-Uncensored-Q8_0.gguf"}, + {"OpenAI-20B-NEO-HRRPlus-Uncensored-IQ4_NL.gguf"}, + {"OpenAI-20B-NEO-Uncensored2-IQ4_NL.gguf"}, + {"OpenAI-20B-NEO-Uncensored2-Q5_1.gguf"}, + {"OpenAI-20B-NEOPlus-Uncensored-IQ4_NL.gguf"}, + {"OpenAI-20B-NEOPlus-Uncensored-Q5_1.gguf"}, + {"OpenAI-20B-NEOPlus-Uncensored-Q8_0.gguf"}}, + { + {"", "OpenAI-20B-NEO-CODE-DI-Uncensored-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "OpenAI-20B-NEO-CODE-DI-Uncensored-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q5_1", "OpenAI-20B-NEO-CODE-DI-Uncensored-Q5_1.gguf", 1, "", "", "", "", ""}, + {"IQ4_NL", "OpenAI-20B-NEO-CODEPlus-Uncensored-IQ4_NL.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_1", "IQ4_NL"}}, + + {"DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF", + {{"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-IQ2_M.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-IQ2_M.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_S.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_S.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q8_0.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-Q8_0.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf"}, + {"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-TOOLS-NEO-MAX-MTP-Q8_0.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"IQ2_M", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-IQ2_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_0", "BF16", "Q4_K_S", "Q4_K_M", "IQ2_M"}}, + + {"DavidAU/Qwen3.6-27B-Heretic-Uncensored-FINETUNE-NEO-CODE-Di-IMatrix-MAX-GGUF", + {{"Qwen3.6-27B-NEO-CODE-HERE-2T-OT-HIGH-Q8_0.gguf"}, + {"Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q4_K_M.gguf"}, + {"Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q6_K.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.6-27B-NEO-CODE-HERE-2T-OT-HIGH-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q6_K", "Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q6_K.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q6_K", "Q4_K_M", "Q8_0"}}, + + {"dcostenco/smollm2-360m-aac-watch", + {{"smollm2-360m-aac-q3ks.gguf"}}, + { + {"", "smollm2-360m-aac-q3ks.gguf", 1, "", "", "", "", ""}, + {"Q3KS", "smollm2-360m-aac-q3ks.gguf", 1, "", "", "", "", ""}}, + {"Q3KS"}}, + + {"DeepBeepMeep/LTX-2", + {{"ltx-2.3-22b-distilled-Q4_K_M_light.gguf"}, + {"ltx-2.3-22b-distilled-Q6_K_light.gguf"}, + {"ltx-2.3-22b-distilled-Q8_0_light.gguf"}}, + { + {"", "ltx-2.3-22b-distilled-Q4_K_M_light.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M_LIGHT", "ltx-2.3-22b-distilled-Q4_K_M_light.gguf", 1, "", "", "", "", ""}, + {"Q8_0_LIGHT", "ltx-2.3-22b-distilled-Q8_0_light.gguf", 1, "", "", "", "", ""}}, + {"Q8_0_LIGHT", "Q6_K_LIGHT", "Q4_K_M_LIGHT"}}, + + {"defog/sqlcoder-7b-2", + {{"sqlcoder-7b-q5_k_m.gguf"}}, + { + {"", "sqlcoder-7b-q5_k_m.gguf", 1, "", "", "", "", ""}, + {"Q5_K_M", "sqlcoder-7b-q5_k_m.gguf", 1, "", "", "", "", ""}}, + {"Q5_K_M"}}, + + {"dgflash/FLUX.1-dev-4bit-gguf", + {{"flux1-dev-Q4_K_S.gguf"}}, + { + {"", "flux1-dev-Q4_K_S.gguf", 1, "", "", "", "", ""}, + {"Q4_K_S", "flux1-dev-Q4_K_S.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_S"}}, + + {"empero-ai/Qwythos-27B-v1-GGUF", + {{"Qwythos-27B-BF16.gguf"}, + {"Qwythos-27B-MTP-BF16.gguf"}, + {"Qwythos-27B-MTP-Q4_K_M.gguf"}, + {"Qwythos-27B-MTP-Q5_K_M.gguf"}, + {"Qwythos-27B-MTP-Q8_0.gguf"}, + {"Qwythos-27B-Q4_K_M.gguf"}, + {"Qwythos-27B-Q5_K_M.gguf"}, + {"Qwythos-27B-Q8_0.gguf"}, + {"mmproj-Qwythos-27B-F16.gguf"}}, + { + {"", "Qwythos-27B-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwythos-27B-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, + {"Q8_0", "Qwythos-27B-MTP-Q8_0.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, + {"BF16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, + {"F16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, + + {"empero-ai/Qwythos-9B-v2-GGUF", + {{"Qwythos-9B-v2-BF16.gguf"}, + {"Qwythos-9B-v2-MTP-BF16.gguf"}, + {"Qwythos-9B-v2-MTP-Q4_K_M.gguf"}, + {"Qwythos-9B-v2-MTP-Q5_K_M.gguf"}, + {"Qwythos-9B-v2-MTP-Q8_0.gguf"}, + {"Qwythos-9B-v2-Q4_K_M.gguf"}, + {"Qwythos-9B-v2-Q5_K_M.gguf"}, + {"Qwythos-9B-v2-Q8_0.gguf"}, + {"mmproj-Qwythos-9B-v2-BF16.gguf"}}, + { + {"", "Qwythos-9B-v2-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-9B-v2-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwythos-9B-v2-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-9B-v2-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwythos-9B-v2-MTP-Q8_0.gguf", 1, "mmproj-Qwythos-9B-v2-BF16.gguf", "", "", "", ""}, + {"BF16", "Qwythos-9B-v2-BF16.gguf", 1, "mmproj-Qwythos-9B-v2-BF16.gguf", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, + + {"FINAL-Bench/POCKET-35B-GGUF", + {{"POCKET-35B-IQ1_M.gguf"}, + {"POCKET-35B-Q3_K_M.gguf"}, + {"POCKET-35B-Q4_K_M.gguf"}}, + { + {"", "POCKET-35B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "POCKET-35B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"IQ1_M", "POCKET-35B-IQ1_M.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M", "Q3_K_M", "IQ1_M"}}, + + {"FINAL-Bench/POCKET-Darwin-180B-GGUF", + {{"POCKET-Darwin-180B-UD-Q4_K_XL", 4}}, + { + {"", "POCKET-Darwin-180B-UD-Q4_K_XL-00001-of-00004.gguf", 4, "", "", "", "", ""}, + {"UD-Q4_K_XL", "POCKET-Darwin-180B-UD-Q4_K_XL-00001-of-00004.gguf", 4, "", "", "", "", ""}}, + {"Q4_K_XL"}}, + + {"FunAudioLLM/Fun-ASR-Nano-2512-GGUF", + {{"fun-asr-nano-2512-f16.gguf"}, + {"fun-asr-nano-2512-q8_0.gguf"}}, + { + {"", "fun-asr-nano-2512-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "fun-asr-nano-2512-q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "fun-asr-nano-2512-f16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "F16"}}, + + {"GCSA-AiLab/GLM-5.3-Flash-Uncensored-RCO-GSQ-GGUF", + {{"Q4/GLM-5.3-Flash-Uncensored-GSQ-RCO-Q4.gguf"}, + {"Q8/GLM-5.3-Flash-Uncensored-Q8_0.gguf"}}, + { + {"", "Q8/GLM-5.3-Flash-Uncensored-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Q8/GLM-5.3-Flash-Uncensored-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q4", "Q4/GLM-5.3-Flash-Uncensored-GSQ-RCO-Q4.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4"}}, + + {"ggml-org/embeddinggemma-2-GGUF", + {{"embeddinggemma-2-BF16.gguf"}, + {"embeddinggemma-2-Q8_0.gguf"}, + {"mmproj-embeddinggemma-2-BF16.gguf"}, + {"mmproj-embeddinggemma-2-Q8_0.gguf"}}, + { + {"", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-embeddinggemma-2-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-embeddinggemma-2-Q8_0.gguf", "", "", "", ""}, + {"BF16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-embeddinggemma-2-BF16.gguf", "", "", "", ""}}, + {"Q8_0", "BF16"}}, + + {"ggml-org/gemma-3-4b-it-GGUF", + {{"gemma-3-4b-it-Q4_K_M.gguf"}, + {"gemma-3-4b-it-Q8_0.gguf"}, + {"gemma-3-4b-it-f16.gguf"}, + {"mmproj-model-f16.gguf"}}, + { + {"", "gemma-3-4b-it-Q4_K_M.gguf", 1, "mmproj-model-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "gemma-3-4b-it-Q4_K_M.gguf", 1, "mmproj-model-f16.gguf", "", "", "", ""}, + {"Q8_0", "gemma-3-4b-it-Q8_0.gguf", 1, "mmproj-model-f16.gguf", "", "", "", ""}, + {"F16", "gemma-3-4b-it-f16.gguf", 1, "mmproj-model-f16.gguf", "", "", "", ""}}, + {"F16", "Q8_0", "Q4_K_M"}}, + + {"ggml-org/gemma-4-12B-it-GGUF", + {{"gemma-4-12B-it-BF16.gguf"}, + {"gemma-4-12B-it-Q4_0.gguf"}, + {"gemma-4-12B-it-Q8_0.gguf"}, + {"mmproj-gemma-4-12B-it-BF16.gguf"}, + {"mmproj-gemma-4-12B-it-Q8_0.gguf"}, + {"mtp-gemma-4-12B-it-BF16.gguf"}, + {"mtp-gemma-4-12B-it-Q4_0.gguf"}, + {"mtp-gemma-4-12B-it-Q8_0.gguf"}}, + { + {"", "gemma-4-12B-it-Q8_0.gguf", 1, "mmproj-gemma-4-12B-it-Q8_0.gguf", "mtp-gemma-4-12B-it-Q8_0.gguf", "", "", ""}, + {"Q8_0", "gemma-4-12B-it-Q8_0.gguf", 1, "mmproj-gemma-4-12B-it-Q8_0.gguf", "mtp-gemma-4-12B-it-Q8_0.gguf", "", "", ""}, + {"BF16", "gemma-4-12B-it-BF16.gguf", 1, "mmproj-gemma-4-12B-it-BF16.gguf", "mtp-gemma-4-12B-it-BF16.gguf", "", "", ""}}, + {"Q8_0", "Q4_0", "BF16"}}, + + {"ggml-org/gemma-4-26B-A4B-it-GGUF", + {{"dflash-gemma-4-26B-A4B-it-BF16.gguf"}, + {"dflash-gemma-4-26B-A4B-it-Q8_0.gguf"}, + {"gemma-4-26B-A4B-it-BF16.gguf"}, + {"gemma-4-26B-A4B-it-Q4_0.gguf"}, + {"gemma-4-26B-A4B-it-Q8_0.gguf"}, + {"mmproj-gemma-4-26B-A4B-it-BF16.gguf"}, + {"mmproj-gemma-4-26B-A4B-it-Q8_0.gguf"}, + {"mtp-gemma-4-26B-A4B-it-BF16.gguf"}, + {"mtp-gemma-4-26B-A4B-it-Q4_0.gguf"}, + {"mtp-gemma-4-26B-A4B-it-Q8_0.gguf"}}, + { + {"", "gemma-4-26B-A4B-it-Q8_0.gguf", 1, "mmproj-gemma-4-26B-A4B-it-Q8_0.gguf", "mtp-gemma-4-26B-A4B-it-Q8_0.gguf", "dflash-gemma-4-26B-A4B-it-Q8_0.gguf", "", ""}, + {"Q8_0", "gemma-4-26B-A4B-it-Q8_0.gguf", 1, "mmproj-gemma-4-26B-A4B-it-Q8_0.gguf", "mtp-gemma-4-26B-A4B-it-Q8_0.gguf", "dflash-gemma-4-26B-A4B-it-Q8_0.gguf", "", ""}, + {"BF16", "gemma-4-26B-A4B-it-BF16.gguf", 1, "mmproj-gemma-4-26B-A4B-it-BF16.gguf", "mtp-gemma-4-26B-A4B-it-BF16.gguf", "dflash-gemma-4-26B-A4B-it-BF16.gguf", "", ""}}, + {"Q8_0", "Q4_0", "BF16"}}, + + {"ggml-org/gemma-4-E2B-it-GGUF", + {{"gemma-4-E2B-it-BF16.gguf"}, + {"gemma-4-E2B-it-Q4_0.gguf"}, + {"gemma-4-E2B-it-Q8_0.gguf"}, + {"mmproj-gemma-4-E2B-it-BF16.gguf"}, + {"mmproj-gemma-4-E2B-it-Q8_0.gguf"}, + {"mtp-gemma-4-E2B-it-BF16.gguf"}, + {"mtp-gemma-4-E2B-it-Q4_0.gguf"}, + {"mtp-gemma-4-E2B-it-Q8_0.gguf"}}, + { + {"", "gemma-4-E2B-it-Q8_0.gguf", 1, "mmproj-gemma-4-E2B-it-Q8_0.gguf", "mtp-gemma-4-E2B-it-Q8_0.gguf", "", "", ""}, + {"Q8_0", "gemma-4-E2B-it-Q8_0.gguf", 1, "mmproj-gemma-4-E2B-it-Q8_0.gguf", "mtp-gemma-4-E2B-it-Q8_0.gguf", "", "", ""}, + {"BF16", "gemma-4-E2B-it-BF16.gguf", 1, "mmproj-gemma-4-E2B-it-BF16.gguf", "mtp-gemma-4-E2B-it-BF16.gguf", "", "", ""}}, + {"Q8_0", "Q4_0", "BF16"}}, + + {"ggml-org/GLM-4.7-Flash-GGUF", + {{"GLM-4.7-Flash-Q4_K.gguf"}, + {"GLM-4.7-Flash-Q8_0.gguf"}}, + { + {"", "GLM-4.7-Flash-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "GLM-4.7-Flash-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q4_K", "GLM-4.7-Flash-Q4_K.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K"}}, + + {"ggml-org/GLM-OCR-GGUF", + {{"GLM-OCR-Q8_0.gguf"}, + {"GLM-OCR-f16.gguf"}, + {"mmproj-GLM-OCR-Q8_0.gguf"}}, + { + {"", "GLM-OCR-Q8_0.gguf", 1, "mmproj-GLM-OCR-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "GLM-OCR-Q8_0.gguf", 1, "mmproj-GLM-OCR-Q8_0.gguf", "", "", "", ""}, + {"F16", "GLM-OCR-f16.gguf", 1, "mmproj-GLM-OCR-Q8_0.gguf", "", "", "", ""}}, + {"F16", "Q8_0"}}, + + {"ggml-org/gpt-oss-120b-GGUF", + {{"eagle3-gpt-oss-120b-BF16.gguf"}, + {"eagle3-gpt-oss-120b-Q8_0.gguf"}, + {"gpt-oss-120b-MXFP4.gguf"}}, + { + {"", "gpt-oss-120b-MXFP4.gguf", 1, "", "", "", "eagle3-gpt-oss-120b-Q8_0.gguf", ""}, + {"Q8_0", "", 0, "", "", "", "eagle3-gpt-oss-120b-Q8_0.gguf", ""}, + {"MXFP4", "gpt-oss-120b-MXFP4.gguf", 1, "", "", "", "eagle3-gpt-oss-120b-Q8_0.gguf", ""}, + {"BF16", "", 0, "", "", "", "eagle3-gpt-oss-120b-BF16.gguf", ""}}, + {"MXFP4"}}, + + {"ggml-org/MiMo-V2.6-Distill-Qwen-9B-GGUF", + {{"MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}, + {"mmproj-MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}}, + { + {"", "MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf", 1, "mmproj-MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf", 1, "mmproj-MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf", "", "", "", ""}}, + {"Q8_0"}}, + + {"ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", + {{"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf"}, + {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q4_0.gguf"}, + {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf"}, + {"mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf"}, + {"mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q4_0.gguf"}, + {"mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf"}}, + { + {"", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", 1, "", "mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", "", "", ""}, + {"Q8_0", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", 1, "", "mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", "", "", ""}, + {"BF16", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf", 1, "", "mtp-NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf", "", "", ""}}, + {"Q8_0", "Q4_0", "BF16"}}, + + {"ggml-org/Qwen2.5-Omni-3B-GGUF", + {{"Qwen2.5-Omni-3B-Q4_K_M.gguf"}, + {"Qwen2.5-Omni-3B-Q8_0.gguf"}, + {"Qwen2.5-Omni-3B-f16.gguf"}, + {"mmproj-Qwen2.5-Omni-3B-Q8_0.gguf"}, + {"mmproj-Qwen2.5-Omni-3B-f16.gguf"}}, + { + {"", "Qwen2.5-Omni-3B-Q4_K_M.gguf", 1, "mmproj-Qwen2.5-Omni-3B-Q8_0.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen2.5-Omni-3B-Q4_K_M.gguf", 1, "mmproj-Qwen2.5-Omni-3B-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "Qwen2.5-Omni-3B-Q8_0.gguf", 1, "mmproj-Qwen2.5-Omni-3B-Q8_0.gguf", "", "", "", ""}, + {"F16", "Qwen2.5-Omni-3B-f16.gguf", 1, "mmproj-Qwen2.5-Omni-3B-f16.gguf", "", "", "", ""}}, + {"F16", "Q8_0", "Q4_K_M"}}, + + {"ggml-org/Qwen2.5-VL-32B-Instruct-GGUF", + {{"Qwen2.5-VL-32B-Instruct-Q4_K_M.gguf"}, + {"Qwen2.5-VL-32B-Instruct-Q8_0.gguf"}, + {"Qwen2.5-VL-32B-Instruct-f16", 2}, + {"mmproj-Qwen2.5-VL-32B-Instruct-Q8_0.gguf"}, + {"mmproj-Qwen2.5-VL-32B-Instruct-f16.gguf"}}, + { + {"", "Qwen2.5-VL-32B-Instruct-Q4_K_M.gguf", 1, "mmproj-Qwen2.5-VL-32B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen2.5-VL-32B-Instruct-Q4_K_M.gguf", 1, "mmproj-Qwen2.5-VL-32B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "Qwen2.5-VL-32B-Instruct-Q8_0.gguf", 1, "mmproj-Qwen2.5-VL-32B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"F16", "Qwen2.5-VL-32B-Instruct-f16-00001-of-00002.gguf", 2, "mmproj-Qwen2.5-VL-32B-Instruct-f16.gguf", "", "", "", ""}}, + {"F16", "Q8_0", "Q4_K_M"}}, + + {"ggml-org/Qwen3-1.7B-GGUF", + {{"Qwen3-1.7B-Q4_K_M.gguf"}, + {"Qwen3-1.7B-Q8_0.gguf"}, + {"Qwen3-1.7B-f16.gguf"}}, + { + {"", "Qwen3-1.7B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen3-1.7B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Qwen3-1.7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "Qwen3-1.7B-f16.gguf", 1, "", "", "", "", ""}}, + {"F16", "Q8_0", "Q4_K_M"}}, + + {"ggml-org/Qwen3-ASR-1.7B-GGUF", + {{"Qwen3-ASR-1.7B-Q8_0.gguf"}, + {"Qwen3-ASR-1.7B-bf16.gguf"}, + {"mmproj-Qwen3-ASR-1.7B-Q8_0.gguf"}, + {"mmproj-Qwen3-ASR-1.7B-bf16.gguf"}}, + { + {"", "Qwen3-ASR-1.7B-Q8_0.gguf", 1, "mmproj-Qwen3-ASR-1.7B-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3-ASR-1.7B-Q8_0.gguf", 1, "mmproj-Qwen3-ASR-1.7B-Q8_0.gguf", "", "", "", ""}, + {"BF16", "Qwen3-ASR-1.7B-bf16.gguf", 1, "mmproj-Qwen3-ASR-1.7B-bf16.gguf", "", "", "", ""}}, + {"BF16", "Q8_0"}}, + + {"ggml-org/Qwen3.6-27B-GGUF", + {{"Qwen3.6-27B-BF16.gguf"}, + {"Qwen3.6-27B-Q4_K_M.gguf"}, + {"Qwen3.6-27B-Q8_0.gguf"}, + {"dflash-Qwen3.6-27B-BF16.gguf"}, + {"dflash-Qwen3.6-27B-Q8_0.gguf"}, + {"mmproj-Qwen3.6-27B-BF16.gguf"}, + {"mmproj-Qwen3.6-27B-Q8_0.gguf"}, + {"mtp-Qwen3.6-27B-BF16.gguf"}, + {"mtp-Qwen3.6-27B-Q4_0.gguf"}, + {"mtp-Qwen3.6-27B-Q8_0.gguf"}}, + { + {"", "Qwen3.6-27B-Q4_K_M.gguf", 1, "mmproj-Qwen3.6-27B-Q8_0.gguf", "mtp-Qwen3.6-27B-Q4_0.gguf", "dflash-Qwen3.6-27B-Q8_0.gguf", "", ""}, + {"Q4_K_M", "Qwen3.6-27B-Q4_K_M.gguf", 1, "mmproj-Qwen3.6-27B-Q8_0.gguf", "mtp-Qwen3.6-27B-Q4_0.gguf", "dflash-Qwen3.6-27B-Q8_0.gguf", "", ""}, + {"Q8_0", "Qwen3.6-27B-Q8_0.gguf", 1, "mmproj-Qwen3.6-27B-Q8_0.gguf", "mtp-Qwen3.6-27B-Q8_0.gguf", "dflash-Qwen3.6-27B-Q8_0.gguf", "", ""}, + {"BF16", "Qwen3.6-27B-BF16.gguf", 1, "mmproj-Qwen3.6-27B-BF16.gguf", "mtp-Qwen3.6-27B-BF16.gguf", "dflash-Qwen3.6-27B-BF16.gguf", "", ""}, + {"Q4_0", "", 0, "", "mtp-Qwen3.6-27B-Q4_0.gguf", "dflash-Qwen3.6-27B-Q8_0.gguf", "", ""}}, + {"Q8_0", "Q4_K_M", "BF16"}}, + + {"ggml-org/Qwen3.8-27B-GGUF", + {{"Qwen3.8-27B-BF16.gguf"}, + {"Qwen3.8-27B-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Q8_0.gguf"}, + {"dflash-Qwen3.8-27B-BF16.gguf"}, + {"dflash-Qwen3.8-27B-Q4_0.gguf"}, + {"dflash-Qwen3.8-27B-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-BF16.gguf"}, + {"mmproj-Qwen3.8-27B-Q8_0.gguf"}, + {"mtp-Qwen3.8-27B-BF16.gguf"}, + {"mtp-Qwen3.8-27B-Q4_0.gguf"}, + {"mtp-Qwen3.8-27B-Q8_0.gguf"}}, + { + {"", "Qwen3.8-27B-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Q8_0.gguf", "mtp-Qwen3.8-27B-Q4_0.gguf", "dflash-Qwen3.8-27B-Q4_0.gguf", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Q8_0.gguf", "mtp-Qwen3.8-27B-Q4_0.gguf", "dflash-Qwen3.8-27B-Q4_0.gguf", "", ""}, + {"Q8_0", "Qwen3.8-27B-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-Q8_0.gguf", "mtp-Qwen3.8-27B-Q8_0.gguf", "dflash-Qwen3.8-27B-Q8_0.gguf", "", ""}, + {"BF16", "Qwen3.8-27B-BF16.gguf", 1, "mmproj-Qwen3.8-27B-BF16.gguf", "mtp-Qwen3.8-27B-BF16.gguf", "dflash-Qwen3.8-27B-BF16.gguf", "", ""}, + {"Q4_0", "", 0, "", "mtp-Qwen3.8-27B-Q4_0.gguf", "dflash-Qwen3.8-27B-Q4_0.gguf", "", ""}}, + {"Q8_0", "Q4_K_M", "BF16"}}, + + {"ggml-org/Qwen3.8-Flash-Next-GGUF", + {{"Qwen3.8-Flash-Next-IQ4_NL", 2}, + {"Qwen3.8-Flash-Next-Q2_K", 2}, + {"Qwen3.8-Flash-Next-Q8_0", 2}, + {"mmproj-Qwen3.8-Flash-Next-Q8_0.gguf"}, + {"mtp-Qwen3.8-Flash-Next-BF16.gguf"}, + {"mtp-Qwen3.8-Flash-Next-Q4_0.gguf"}, + {"mtp-Qwen3.8-Flash-Next-Q8_0.gguf"}}, + { + {"", "Qwen3.8-Flash-Next-Q8_0-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-Q8_0.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"Q8_0", "Qwen3.8-Flash-Next-Q8_0-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-Q8_0.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"IQ4_NL", "Qwen3.8-Flash-Next-IQ4_NL-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-Q8_0.gguf", "mtp-Qwen3.8-Flash-Next-Q4_0.gguf", "", "", ""}, + {"BF16", "", 0, "", "mtp-Qwen3.8-Flash-Next-BF16.gguf", "", "", ""}}, + {"Q8_0", "Q2_K", "IQ4_NL"}}, + + {"ggml-org/SmolVLM-500M-Instruct-GGUF", + {{"SmolVLM-500M-Instruct-Q8_0.gguf"}, + {"SmolVLM-500M-Instruct-f16.gguf"}, + {"mmproj-SmolVLM-500M-Instruct-Q8_0.gguf"}, + {"mmproj-SmolVLM-500M-Instruct-f16.gguf"}}, + { + {"", "SmolVLM-500M-Instruct-Q8_0.gguf", 1, "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "SmolVLM-500M-Instruct-Q8_0.gguf", 1, "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf", "", "", "", ""}, + {"F16", "SmolVLM-500M-Instruct-f16.gguf", 1, "mmproj-SmolVLM-500M-Instruct-f16.gguf", "", "", "", ""}}, + {"F16", "Q8_0"}}, + + {"ggml-org/test-model-stories260K", + {{"stories260K-f32.gguf"}}, + { + {"", "stories260K-f32.gguf", 1, "", "", "", "", ""}, + {"F32", "stories260K-f32.gguf", 1, "", "", "", "", ""}}, + {"F32"}}, + + {"GnLOLot/MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-GGUF", + {{"MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-F16.gguf"}, + {"MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-Q8_0.gguf"}}, + { + {"", "MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-F16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "F16"}}, + + {"google/gemma-4-12B-it-qat-q4_0-gguf", + {{"gemma-4-12b-it-qat-q4_0.gguf"}, + {"mmproj-gemma-4-12b-it-qat-q4_0.gguf"}}, + { + {"", "gemma-4-12b-it-qat-q4_0.gguf", 1, "mmproj-gemma-4-12b-it-qat-q4_0.gguf", "", "", "", ""}, + {"Q4_0", "gemma-4-12b-it-qat-q4_0.gguf", 1, "mmproj-gemma-4-12b-it-qat-q4_0.gguf", "", "", "", ""}}, + {"Q4_0"}}, + + {"handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf", + {{"bundle/multitalker-parakeet-streaming-0.6b-v1-F16.gguf"}, + {"bundle/multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf"}, + {"bundle/multitalker-parakeet-streaming-0.6b-v1-Q5_K_M.gguf"}, + {"bundle/multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf"}}, + { + {"", "bundle/multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "bundle/multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "bundle/multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "bundle/multitalker-parakeet-streaming-0.6b-v1-F16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "F16"}}, + + {"HauhauCS/Gemma-4-E2B-Uncensored-HauhauCS-Aggressive", + {{"Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf"}, + {"Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf"}, + {"Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"}, + {"mmproj-Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-f16.gguf"}}, + { + {"", "Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf", 1, "mmproj-Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"IQ3_M", "Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf", 1, "mmproj-Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q8_K_P", "Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf", 1, "mmproj-Gemma-4-E2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_K_P", "Q4_K_P", "IQ3_M"}}, + + {"HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive", + {{"Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf"}, + {"Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"}, + {"Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"}, + {"mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf"}}, + { + {"", "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"IQ3_M", "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf", 1, "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q8_K_P", "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf", 1, "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_K_P", "Q4_K_M", "IQ3_M"}}, + + {"HauhauCS/GPTOSS-120B-Uncensored-HauhauCS-Aggressive", + {{"GPTOSS-120B-Uncensored-HauhauCS-Aggressive-MXFP4.gguf"}}, + { + {"", "GPTOSS-120B-Uncensored-HauhauCS-Aggressive-MXFP4.gguf", 1, "", "", "", "", ""}, + {"MXFP4", "GPTOSS-120B-Uncensored-HauhauCS-Aggressive-MXFP4.gguf", 1, "", "", "", "", ""}}, + {"MXFP4"}}, + + {"HauhauCS/Qwen3.5-27B-Uncensored-HauhauCS-Aggressive", + {{"Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf"}, + {"Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q3_K_M.gguf"}, + {"Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"}, + {"Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf"}, + {"mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"}}, + { + {"", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"BF16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"F16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_M", "BF16"}}, + + {"HauhauCS/Qwen3.5-2B-Uncensored-HauhauCS-Aggressive", + {{"Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf"}, + {"Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"}, + {"Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf"}, + {"Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf"}, + {"mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf"}}, + { + {"", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"BF16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, + {"F16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"HauhauCS/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive", + {{"Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf"}, + {"Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"}, + {"Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf"}, + {"Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf"}, + {"mmproj-Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf"}}, + { + {"", "Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf", "", "", "", ""}, + {"BF16", "Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-4B-Uncensored-HauhauCS-Aggressive-BF16.gguf", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"Hikari07jp/Ternary-Bonsai-27B-Abliterated-LowDeg-GGUF", + {{"Ternary-Bonsai-27B-Abliterated-LowDeg-Q2_0.gguf"}}, + { + {"", "Ternary-Bonsai-27B-Abliterated-LowDeg-Q2_0.gguf", 1, "", "", "", "", ""}, + {"Q2_0", "Ternary-Bonsai-27B-Abliterated-LowDeg-Q2_0.gguf", 1, "", "", "", "", ""}}, + {"Q2_0"}}, + + {"Hob-forge/Kolibri-1-GGUF", + {{"Kolibri-1-Q2_K.gguf"}, + {"Kolibri-1-Q4_K_M.gguf"}, + {"Q5_K_M/Kolibri-1-Q5_K_M", 2}, + {"Q8_0/Kolibri-1-Q8_0", 2}}, + { + {"", "Kolibri-1-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Kolibri-1-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Q8_0/Kolibri-1-Q8_0-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"Q2_K", "Kolibri-1-Q2_K.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "Q2_K"}}, + + {"huihui-ai/Huihui-Agents-A1-abliterated-GGUF", + {{"Agents-A1-abliterated-Q4_K.gguf"}, + {"Agents-A1-abliterated-Q8_0.gguf"}, + {"Agents-A1-abliterated-bf16.gguf"}, + {"mmproj-model-bf16.gguf"}}, + { + {"", "Agents-A1-abliterated-Q8_0.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"Q8_0", "Agents-A1-abliterated-Q8_0.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"Q4_K", "Agents-A1-abliterated-Q4_K.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"BF16", "Agents-A1-abliterated-bf16.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}}, + {"BF16", "Q8_0", "Q4_K"}}, + + {"huihui-ai/Huihui-GLM-5.3-Flash-abliterated-GGUF", + {{"UD-IQ1_M/GLM-5.3-Flash-UD-IQ1_M", 3}, + {"UD-Q2_K_XL/GLM-5.3-Flash-UD-Q2_K_XL", 4}, + {"UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL", 7}, + {"mmproj-model-bf16.gguf"}}, + { + {"", "UD-IQ1_M/GLM-5.3-Flash-UD-IQ1_M-00001-of-00003.gguf", 3, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"UD-IQ1_M", "UD-IQ1_M/GLM-5.3-Flash-UD-IQ1_M-00001-of-00003.gguf", 3, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"UD-Q6_K_XL", "UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf", 7, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q6_K_XL", "Q2_K_XL", "IQ1_M"}}, + + {"huihui-ai/Huihui-Qwen3.8-Flash-Next-abliterated-GGUF", + {{"BF16/Qwen3.8-Flash-Next-BF16", 8}, + {"Q8_0/Qwen3.8-Flash-Next-Q8_0", 6}, + {"mmproj-Qwen3.8-Flash-Next-BF16.gguf"}, + {"mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf"}, + {"mmproj-model-bf16.gguf"}, + {"swift-IQ3_XXS/Swift-Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS", 2}, + {"unsloth-UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL", 4}}, + { + {"", "Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf", 6, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf", 6, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3.8-Flash-Next-BF16-00001-of-00008.gguf", 8, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"UD-Q4_K_XL", "unsloth-UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf", 4, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}}, + {"Q4_K_XL", "IQ3_XXS", "Q8_0", "BF16"}}, + + {"huihui-ai/Qwen2.5-VL-3B-Instruct-abliterated", + {{"GGUF/ggml-model-Q4_K_M.gguf"}, + {"GGUF/ggml-model-f16.gguf"}, + {"GGUF/ggml-model-q8_0.gguf"}, + {"GGUF/mmproj-ggml-model-f16.gguf"}}, + { + {"", "GGUF/ggml-model-Q4_K_M.gguf", 1, "GGUF/mmproj-ggml-model-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "GGUF/ggml-model-Q4_K_M.gguf", 1, "GGUF/mmproj-ggml-model-f16.gguf", "", "", "", ""}, + {"Q8_0", "GGUF/ggml-model-q8_0.gguf", 1, "GGUF/mmproj-ggml-model-f16.gguf", "", "", "", ""}}, + {"Q8_0", "F16", "Q4_K_M"}}, + + {"ibm-granite/granite-4.2-30b-GGUF", + {{"granite-4.2-30b-Q2_K.gguf"}, + {"granite-4.2-30b-Q4_K_M.gguf"}, + {"granite-4.2-30b-Q4_K_S.gguf"}, + {"granite-4.2-30b-Q8_0.gguf"}, + {"granite-4.2-30b-bf16", 2}}, + { + {"", "granite-4.2-30b-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "granite-4.2-30b-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "granite-4.2-30b-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q2_K", "granite-4.2-30b-Q2_K.gguf", 1, "", "", "", "", ""}, + {"BF16", "granite-4.2-30b-bf16-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"BF16", "Q8_0", "Q4_K_S", "Q4_K_M", "Q2_K"}}, + + {"Isi99999/LTX-Video", + {{"LTX_2_3/ltx-2.3-22b-dev-Q4_K_M.gguf"}}, + { + {"", "LTX_2_3/ltx-2.3-22b-dev-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "LTX_2_3/ltx-2.3-22b-dev-Q4_K_M.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF", + {{"IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M", 2}, + {"mmproj-Qwen3.8-Flash-Next-BF16.gguf"}}, + { + {"", "IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"IQ1_M", "IQ1_M/Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"IQ1_M"}}, + + {"ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF", + {{"IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS", 2}, + {"IQ3_XXS/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS", 2}, + {"Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0", 2}, + {"mmproj-Qwen3.8-Flash-Next-BF16.gguf"}}, + { + {"", "IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"IQ2_XS", "IQ2_XS/Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"Q2_0", "Q2_0/Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q2_0", "IQ3_XXS", "IQ2_XS"}}, + + {"Jackrong/Qwopus3.6-27B-v1-preview-GGUF", + {{"Qwopus3.6-27B-v1-preview-Q4_K_M.gguf"}, + {"Qwopus3.6-27B-v1-preview-Q6_K.gguf"}, + {"Qwopus3.6-27B-v1-preview-Q8_0.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwopus3.6-27B-v1-preview-Q4_K_M.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwopus3.6-27B-v1-preview-Q4_K_M.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"Q8_0", "Qwopus3.6-27B-v1-preview-Q8_0.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"F32", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M"}}, + + {"Jackrong/Qwopus3.8-27B-Flash-V2-GGUF", + {{"Qwopus3.8-27B-Flash-V2-MTP-BF16.gguf"}, + {"Qwopus3.8-27B-Flash-V2-MTP-Q4_K_M.gguf"}, + {"Qwopus3.8-27B-Flash-V2-MTP-Q8_0.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwopus3.8-27B-Flash-V2-MTP-Q4_K_M.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwopus3.8-27B-Flash-V2-MTP-Q4_K_M.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"Q8_0", "Qwopus3.8-27B-Flash-V2-MTP-Q8_0.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"BF16", "Qwopus3.8-27B-Flash-V2-MTP-BF16.gguf", 1, "mmproj-F32.gguf", "", "", "", ""}, + {"F32", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "BF16"}}, + + {"JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q4_K_M", + {{"Mellum2-12B-A2.5B-Instruct-Q4_K_M.gguf"}}, + { + {"", "Mellum2-12B-A2.5B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Mellum2-12B-A2.5B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"JetBrains/Mellum2-12B-A2.5B-Instruct-GGUF-Q8_0", + {{"Mellum2-12B-A2.5B-Instruct-Q8_0.gguf"}}, + { + {"", "Mellum2-12B-A2.5B-Instruct-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Mellum2-12B-A2.5B-Instruct-Q8_0.gguf", 1, "", "", "", "", ""}}, + {"Q8_0"}}, + + {"JetBrains/Mellum2.1-12B-A2.5B-Thinking-GGUF", + {{"Mellum2.1-12B-A2.5B-Thinking-BF16.gguf"}, + {"Mellum2.1-12B-A2.5B-Thinking-Q4_K_M.gguf"}, + {"Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf"}}, + { + {"", "Mellum2.1-12B-A2.5B-Thinking-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Mellum2.1-12B-A2.5B-Thinking-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Mellum2.1-12B-A2.5B-Thinking-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "Mellum2.1-12B-A2.5B-Thinking-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "BF16"}}, + + {"JonathanColetti/Qwen3.8-27B-Uncensored-GGUF", + {{"Qwen3.8-27B-Uncensored-IQ2_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q5_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q8_0.gguf"}, + {"Qwen3.8-27B-Uncensored-draft-Q4_0.gguf"}, + {"Qwen3.8-27B-Uncensored-draft-Q8_0.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-IQ2_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q5_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-noMTP-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-Uncensored-F16.gguf"}}, + { + {"", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-Uncensored-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"IQ2_M", "Qwen3.8-27B-Uncensored-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"Q4_0", "Qwen3.8-27B-Uncensored-draft-Q4_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}, + {"mtp", "Qwen3.8-27B-Uncensored-noMTP-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ2_M", "Q4_0"}}, + + {"kai-os/Carnice-V2-27b-GGUF", + {{"carnice-v2-27b-IQ2_M.gguf"}, + {"carnice-v2-27b-Q4_K_M.gguf"}, + {"carnice-v2-27b-Q5_K_M.gguf"}, + {"carnice-v2-27b-Q8_0.gguf"}, + {"carnice-v2-27b-bf16.gguf"}}, + { + {"", "carnice-v2-27b-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "carnice-v2-27b-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "carnice-v2-27b-Q8_0.gguf", 1, "", "", "", "", ""}, + {"IQ2_M", "carnice-v2-27b-IQ2_M.gguf", 1, "", "", "", "", ""}, + {"BF16", "carnice-v2-27b-bf16.gguf", 1, "", "", "", "", ""}}, + {"BF16", "Q8_0", "Q5_K_M", "Q4_K_M", "IQ2_M"}}, + + {"KBLab/sentence-bert-swedish-cased", + {{"Sentence-Bert-Swedish-Cased-124M-BF16.gguf"}}, + { + {"", "Sentence-Bert-Swedish-Cased-124M-BF16.gguf", 1, "", "", "", "", ""}, + {"BF16", "Sentence-Bert-Swedish-Cased-124M-BF16.gguf", 1, "", "", "", "", ""}}, + {"BF16"}}, + + {"KBlueLeaf/DanTagGen-delta-rev2", + {{"ggml-model-Q6_K.gguf"}, + {"ggml-model-Q8_0.gguf"}, + {"ggml-model-f16.gguf"}}, + { + {"", "ggml-model-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "ggml-model-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q6_K", "ggml-model-Q6_K.gguf", 1, "", "", "", "", ""}, + {"F16", "ggml-model-f16.gguf", 1, "", "", "", "", ""}}, + {"F16", "Q8_0", "Q6_K"}}, + + {"KittenML/kitten-tts-2", + {{"cpp/model-tq2_1.gguf"}}, + { + {"", "cpp/model-tq2_1.gguf", 1, "", "", "", "", ""}, + {"TQ2_1", "cpp/model-tq2_1.gguf", 1, "", "", "", "", ""}}, + {"TQ2_1"}}, + + {"LiquidAI/LFM2.5-2.6B-DSpark-GGUF", + {{"LFM2.5-2.6B-DSpark-F16.gguf"}, + {"LFM2.5-2.6B-DSpark-Q4_K_M.gguf"}, + {"LFM2.5-2.6B-DSpark-Q8_0.gguf"}}, + { + {"", "LFM2.5-2.6B-DSpark-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "LFM2.5-2.6B-DSpark-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "LFM2.5-2.6B-DSpark-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "LFM2.5-2.6B-DSpark-F16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "F16"}}, + + {"LiquidAI/LFM2.5-Audio-1.5B-GGUF", + {{"LFM2.5-Audio-1.5B-F16.gguf"}, + {"LFM2.5-Audio-1.5B-Q4_0.gguf"}, + {"LFM2.5-Audio-1.5B-Q8_0.gguf"}, + {"mmproj-LFM2.5-Audio-1.5B-F16.gguf"}, + {"mmproj-LFM2.5-Audio-1.5B-Q4_0.gguf"}, + {"mmproj-LFM2.5-Audio-1.5B-Q8_0.gguf"}, + {"tokenizer-LFM2.5-Audio-1.5B-F16.gguf"}, + {"tokenizer-LFM2.5-Audio-1.5B-Q4_0.gguf"}, + {"tokenizer-LFM2.5-Audio-1.5B-Q8_0.gguf"}, + {"vocoder-LFM2.5-Audio-1.5B-F16.gguf"}, + {"vocoder-LFM2.5-Audio-1.5B-Q4_0.gguf"}, + {"vocoder-LFM2.5-Audio-1.5B-Q8_0.gguf"}}, + { + {"", "LFM2.5-Audio-1.5B-Q8_0.gguf", 1, "mmproj-LFM2.5-Audio-1.5B-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "LFM2.5-Audio-1.5B-Q8_0.gguf", 1, "mmproj-LFM2.5-Audio-1.5B-Q8_0.gguf", "", "", "", ""}, + {"F16", "LFM2.5-Audio-1.5B-F16.gguf", 1, "mmproj-LFM2.5-Audio-1.5B-F16.gguf", "", "", "", ""}}, + {"Q8_0", "Q4_0", "F16"}}, + + {"llm-jp/llm-jp-4-33b-thinking-gguf", + {{"llm-jp-4-33b-thinking-BF16.gguf"}, + {"llm-jp-4-33b-thinking-Q4_K_M.gguf"}}, + { + {"", "llm-jp-4-33b-thinking-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "llm-jp-4-33b-thinking-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"BF16", "llm-jp-4-33b-thinking-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M", "BF16"}}, + + {"lmg-anon/vntl-llama3-8b-v2-gguf", + {{"vntl-llama3-8b-v2-hf-q5_k_m.gguf"}, + {"vntl-llama3-8b-v2-hf-q8_0.gguf"}}, + { + {"", "vntl-llama3-8b-v2-hf-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "vntl-llama3-8b-v2-hf-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q5_K_M", "vntl-llama3-8b-v2-hf-q5_k_m.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M"}}, + + {"lmstudio-ai/gemma-2b-it-GGUF", + {{"gemma-2b-it-q4_k_m.gguf"}, + {"gemma-2b-it-q8_0.gguf"}}, + { + {"", "gemma-2b-it-q4_k_m.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "gemma-2b-it-q4_k_m.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "gemma-2b-it-q8_0.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M"}}, + + {"lmstudio-community/Bonsai-27B-GGUF", + {{"Bonsai-27B-Q1_0.gguf"}, + {"mmproj-Bonsai-27B-BF16.gguf"}}, + { + {"", "Bonsai-27B-Q1_0.gguf", 1, "mmproj-Bonsai-27B-BF16.gguf", "", "", "", ""}, + {"Q1_0", "Bonsai-27B-Q1_0.gguf", 1, "mmproj-Bonsai-27B-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q1_0"}}, + + {"lmstudio-community/GLM-4.6V-Flash-GGUF", + {{"GLM-4.6V-Flash-Q4_K_M.gguf"}, + {"GLM-4.6V-Flash-Q6_K.gguf"}, + {"GLM-4.6V-Flash-Q8_0.gguf"}, + {"mmproj-GLM-4.6V-Flash-F16.gguf"}}, + { + {"", "GLM-4.6V-Flash-Q4_K_M.gguf", 1, "mmproj-GLM-4.6V-Flash-F16.gguf", "", "", "", ""}, + {"Q4_K_M", "GLM-4.6V-Flash-Q4_K_M.gguf", 1, "mmproj-GLM-4.6V-Flash-F16.gguf", "", "", "", ""}, + {"Q8_0", "GLM-4.6V-Flash-Q8_0.gguf", 1, "mmproj-GLM-4.6V-Flash-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M"}}, + + {"lmstudio-community/gpt-oss-120b-GGUF", + {{"gpt-oss-120b-MXFP4", 2}}, + { + {"", "gpt-oss-120b-MXFP4-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"MXFP4", "gpt-oss-120b-MXFP4-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"MXFP4"}}, + + {"lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF", + {{"Meta-Llama-3.1-8B-Instruct-IQ4_XS.gguf"}, + {"Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"}, + {"Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf"}, + {"Meta-Llama-3.1-8B-Instruct-Q8_0.gguf"}}, + { + {"", "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Meta-Llama-3.1-8B-Instruct-Q8_0.gguf", 1, "", "", "", "", ""}, + {"IQ4_XS", "Meta-Llama-3.1-8B-Instruct-IQ4_XS.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ4_XS"}}, + + {"lmstudio-community/Muse-Glimmer-30B-GGUF", + {{"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf"}, + {"mmproj-Muse-Glimmer-30B-Q4_K_M.gguf"}}, + { + {"", "Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", 1, "mmproj-Muse-Glimmer-30B-Q4_K_M.gguf", "", "", "", ""}, + {"Q4_K_M", "Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", 1, "mmproj-Muse-Glimmer-30B-Q4_K_M.gguf", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"lmstudio-community/Qwen3-Coder-Next-GGUF", + {{"Qwen3-Coder-Next-Q4_K_M.gguf"}, + {"Qwen3-Coder-Next-Q6_K", 2}, + {"Qwen3-Coder-Next-Q8_0", 3}}, + { + {"", "Qwen3-Coder-Next-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen3-Coder-Next-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Qwen3-Coder-Next-Q8_0-00001-of-00003.gguf", 3, "", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M"}}, + + {"LocalAI-io/privacy-filter-multilingual-GGUF", + {{"privacy-filter-multilingual-f16.gguf"}, + {"privacy-filter-multilingual-q8.gguf"}}, + { + {"", "privacy-filter-multilingual-f16.gguf", 1, "", "", "", "", ""}, + {"F16", "privacy-filter-multilingual-f16.gguf", 1, "", "", "", "", ""}, + {"Q8", "privacy-filter-multilingual-q8.gguf", 1, "", "", "", "", ""}}, + {"Q8", "F16"}}, + + {"meta-models/Muse-Glimmer-30B-GGUF", + {{"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf"}, + {"Muse-Glimmer-30B-KQuant-Dynamic-Q4_K_XL.gguf"}, + {"dflash-Muse-Glimmer-30B-Q4_K_M.gguf"}, + {"mmproj-Muse-Glimmer-30B-Q4_K_M.gguf"}}, + { + {"", "Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", 1, "mmproj-Muse-Glimmer-30B-Q4_K_M.gguf", "", "dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "", ""}, + {"Q4_K_M", "Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", 1, "mmproj-Muse-Glimmer-30B-Q4_K_M.gguf", "", "dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "", ""}, + {"Q4_K_XL", "Muse-Glimmer-30B-KQuant-Dynamic-Q4_K_XL.gguf", 1, "mmproj-Muse-Glimmer-30B-Q4_K_M.gguf", "", "dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "", ""}}, + {"Q4_K_XL", "Q4_K_M"}}, + + {"migtissera/Tess-4-27B-GGUF", + {{"Tess-4-27B-Q4_K_M.gguf"}, + {"Tess-4-27B-Q6_K.gguf"}, + {"Tess-4-27B-Q8_0.gguf"}, + {"mmproj-Tess-4-27B-F16.gguf"}, + {"mtp-Tess-4-27B-Q4_K_M.gguf"}, + {"mtp-Tess-4-27B-Q8_0.gguf"}}, + { + {"", "Tess-4-27B-Q4_K_M.gguf", 1, "mmproj-Tess-4-27B-F16.gguf", "mtp-Tess-4-27B-Q4_K_M.gguf", "", "", ""}, + {"Q4_K_M", "Tess-4-27B-Q4_K_M.gguf", 1, "mmproj-Tess-4-27B-F16.gguf", "mtp-Tess-4-27B-Q4_K_M.gguf", "", "", ""}, + {"Q8_0", "Tess-4-27B-Q8_0.gguf", 1, "mmproj-Tess-4-27B-F16.gguf", "mtp-Tess-4-27B-Q8_0.gguf", "", "", ""}, + {"F16", "", 0, "", "mtp-Tess-4-27B-Q8_0.gguf", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M"}}, + + {"mixedbread-ai/mxbai-embed-large-v1", + {{"gguf/mxbai-embed-large-v1-f16.gguf"}}, + { + {"", "gguf/mxbai-embed-large-v1-f16.gguf", 1, "", "", "", "", ""}, + {"F16", "gguf/mxbai-embed-large-v1-f16.gguf", 1, "", "", "", "", ""}}, + {"F16"}}, + + {"mondk/MiniCPM5-2B-Abliterated-Uncensored-GGUF", + {{"minicpm5-2b-abliterated-F16.gguf"}, + {"minicpm5-2b-abliterated-Q4_K_M.gguf"}, + {"minicpm5-2b-abliterated-Q8_0.gguf"}}, + { + {"", "minicpm5-2b-abliterated-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "minicpm5-2b-abliterated-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "minicpm5-2b-abliterated-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "minicpm5-2b-abliterated-F16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "F16"}}, + + {"NANI-Nithin/K2-Horizon-MoVA-36B-A4B-GGUF", + {{"K2-Horizon-MoVA-36B-A4B-BF16.gguf"}, + {"K2-Horizon-MoVA-36B-A4B-Q2_0.gguf"}, + {"K2-Horizon-MoVA-36B-A4B-Q4_K_M.gguf"}, + {"K2-Horizon-MoVA-36B-A4B-Q8_0.gguf"}}, + { + {"", "K2-Horizon-MoVA-36B-A4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "K2-Horizon-MoVA-36B-A4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "K2-Horizon-MoVA-36B-A4B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "K2-Horizon-MoVA-36B-A4B-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q2_0", "BF16"}}, + + {"ngquocvinh/AliceAI-T5-35B-A0.6B-GGUF", + {{"AliceAI-T5-35B-A0.6B-BF16.gguf"}, + {"AliceAI-T5-35B-A0.6B-Q3_K_S.gguf"}, + {"AliceAI-T5-35B-A0.6B-Q4_K_M.gguf"}, + {"AliceAI-T5-35B-A0.6B-Q8_0.gguf"}, + {"AliceAI-T5-35B-A0.6B-TQ2_0.gguf"}}, + { + {"", "AliceAI-T5-35B-A0.6B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "AliceAI-T5-35B-A0.6B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "AliceAI-T5-35B-A0.6B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "AliceAI-T5-35B-A0.6B-BF16.gguf", 1, "", "", "", "", ""}, + {"TQ2_0", "AliceAI-T5-35B-A0.6B-TQ2_0.gguf", 1, "", "", "", "", ""}}, + {"TQ2_0", "Q8_0", "Q4_K_M", "Q3_K_S", "BF16"}}, + + {"ngquocvinh/NeoHorse-1-4B-GGUF", + {{"NeoHorse-1-4B-IQ1_M.gguf"}, + {"NeoHorse-1-4B-Q2_K_S.gguf"}, + {"NeoHorse-1-4B-Q4_K_M.gguf"}, + {"NeoHorse-1-4B-Q8_0.gguf"}, + {"NeoHorse-1-4B-TQ2_0.gguf"}}, + { + {"", "NeoHorse-1-4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "NeoHorse-1-4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "NeoHorse-1-4B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"IQ1_M", "NeoHorse-1-4B-IQ1_M.gguf", 1, "", "", "", "", ""}, + {"TQ2_0", "NeoHorse-1-4B-TQ2_0.gguf", 1, "", "", "", "", ""}}, + {"TQ2_0", "Q8_0", "Q4_K_M", "Q2_K_S", "IQ1_M"}}, + + {"nif0/Qwen3-VL-32B-Instruct-MiniMax-H3-GGUF", + {{"Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-IQ4_XS.gguf"}, + {"Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-Q4_K_M.gguf"}, + {"Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-UD-IQ3_XXS.gguf"}, + {"Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-mmproj-F32.gguf"}}, + { + {"", "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-Q4_K_M.gguf", 1, "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-mmproj-F32.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-Q4_K_M.gguf", 1, "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-mmproj-F32.gguf", "", "", "", ""}, + {"IQ4_XS", "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-IQ4_XS.gguf", 1, "Qwen3-VL-32B-Instruct-MiniMax-H3-L0-49-mmproj-F32.gguf", "", "", "", ""}, + {"F32", "", 0, "", "", "", "", ""}}, + {"IQ3_XXS", "Q4_K_M", "IQ4_XS"}}, + + {"NikolayKozloff/Meta-Llama-3.1-8B-Instruct-Q8_0-GGUF", + {{"meta-llama-3.1-8b-instruct-q8_0.gguf"}}, + { + {"", "meta-llama-3.1-8b-instruct-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "meta-llama-3.1-8b-instruct-q8_0.gguf", 1, "", "", "", "", ""}}, + {"Q8_0"}}, + + {"noctrex/Huihui-Qwen3-VL-4B-Instruct-abliterated-GGUF", + {{"Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf"}, + {"Huihui-Qwen3-VL-4B-Instruct-abliterated-IQ4_XS.gguf"}, + {"Huihui-Qwen3-VL-4B-Instruct-abliterated-Q4_K_M.gguf"}, + {"Huihui-Qwen3-VL-4B-Instruct-abliterated-Q8_0.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "IQ4_XS", "BF16"}}, + + {"OBLITERATUS/Gemma-4-12B-OBLITERATED", + {{"Gemma-4-12B-OBLITERATED-BF16.gguf"}, + {"Gemma-4-12B-OBLITERATED-Q4_K_M.gguf"}, + {"Gemma-4-12B-OBLITERATED-v2-Q8_0.gguf"}}, + { + {"", "Gemma-4-12B-OBLITERATED-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Gemma-4-12B-OBLITERATED-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Gemma-4-12B-OBLITERATED-v2-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "Gemma-4-12B-OBLITERATED-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "BF16"}}, + + {"OBLITERATUS/gemma-4-E4B-it-OBLITERATED", + {{"gemma-4-E4B-it-OBLITERATED-Q4_K_M.gguf"}, + {"gemma-4-E4B-it-OBLITERATED-Q8_0.gguf"}, + {"gemma-4-E4B-it-OBLITERATED-mmproj-f16.gguf"}}, + { + {"", "gemma-4-E4B-it-OBLITERATED-Q4_K_M.gguf", 1, "gemma-4-E4B-it-OBLITERATED-mmproj-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "gemma-4-E4B-it-OBLITERATED-Q4_K_M.gguf", 1, "gemma-4-E4B-it-OBLITERATED-mmproj-f16.gguf", "", "", "", ""}, + {"Q8_0", "gemma-4-E4B-it-OBLITERATED-Q8_0.gguf", 1, "gemma-4-E4B-it-OBLITERATED-mmproj-f16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M"}}, + + {"OBLITERATUS/Qwen3.8-27B-OBLITERATED", + {{"Qwen3.8-27B-OBLITERATED-IQ4_XS.gguf"}, + {"Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf"}, + {"Qwen3.8-27B-OBLITERATED-Q8_0.gguf"}, + {"mmproj-model-bf16.gguf"}}, + { + {"", "Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-OBLITERATED-Q8_0.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"IQ4_XS", "Qwen3.8-27B-OBLITERATED-IQ4_XS.gguf", 1, "mmproj-model-bf16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "IQ4_XS"}}, + + {"openthaigpt/openthaigpt1.5-7b-instruct", + {{"openthaigpt1.5-7B-F16.gguf"}, + {"openthaigpt1.5-7B-instruct-Q3KM.gguf"}, + {"openthaigpt1.5-7B-instruct-Q4KM.gguf"}}, + { + {"", "openthaigpt1.5-7B-F16.gguf", 1, "", "", "", "", ""}, + {"F16", "openthaigpt1.5-7B-F16.gguf", 1, "", "", "", "", ""}, + {"Q4KM", "openthaigpt1.5-7B-instruct-Q4KM.gguf", 1, "", "", "", "", ""}}, + {"Q4KM", "Q3KM", "F16"}}, + + {"orcarouter/Qwen3.8-27B-Uncensored-GGUF", + {{"Qwen3.8-27B-Uncensored-F16", 2}, + {"Qwen3.8-27B-Uncensored-Q3_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-Uncensored-f16.gguf"}}, + { + {"", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Uncensored-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-f16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-Uncensored-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-f16.gguf", "", "", "", ""}, + {"F16", "Qwen3.8-27B-Uncensored-F16-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-27B-Uncensored-f16.gguf", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_M", "F16"}}, + + {"oruk/orukeet", + {{"orukeet-transcribe-cpp-Q8_0.gguf"}, + {"orukeet-v0.1.0-q8.gguf"}, + {"transcribe-cpp/orukeet-Q8_0.gguf"}}, + { + {"", "orukeet-transcribe-cpp-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "orukeet-transcribe-cpp-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8", "orukeet-v0.1.0-q8.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q8"}}, + + {"peculiar-ragdoll/Cyber-Tiel-Coder-35B-A3B-GGUF-MTP", + {{"Cyber-Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf"}, + {"Cyber-Tiel-Coder-35B-A3B-MTP-UD-Q4_K_S.gguf"}, + {"Cyber-Tiel-Coder-35B-A3B-MTP-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-Q8_0.gguf"}}, + { + {"", "Cyber-Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "", 0, "", "", "", "", ""}, + {"UD-IQ3_XXS", "Cyber-Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Cyber-Tiel-Coder-35B-A3B-MTP-UD-Q8_K_XL.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_K_XL", "Q4_K_S", "IQ3_XXS"}}, + + {"peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF", + {{"Dirk-Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf"}, + {"Dirk-Qwen3.8-27B-UD-Q4_K_S.gguf"}, + {"Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf"}, + {"mmproj-F16.gguf"}}, + { + {"", "Dirk-Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"IQ2_S", "Dirk-Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf", 1, "mmproj-F16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q8_K_XL", "Q4_K_S", "IQ2_S"}}, + + {"peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP", + {{"Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf"}, + {"Tiel-Coder-35B-A3B-MTP-UD-Q4_K_S.gguf"}, + {"Tiel-Coder-35B-A3B-MTP-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-IQ3_XXS", "Tiel-Coder-35B-A3B-MTP-UD-IQ3_XXS.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Tiel-Coder-35B-A3B-MTP-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_K_XL", "Q4_K_S", "IQ3_XXS"}}, + + {"perplexity-ai/pplx-computer-qwen-3-8-27b-dflash2-gguf-20260826", + {{"draft/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"}, + {"draft/qwen38-27b-mtp-Q8_0.gguf"}, + {"qwen38-27b-Q4_K_M.gguf"}, + {"qwen38-27b-mmproj-f16.gguf"}}, + { + {"", "draft/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", 1, "qwen38-27b-mmproj-f16.gguf", "draft/qwen38-27b-mtp-Q8_0.gguf", "", "", ""}, + {"Q4_K_M", "draft/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", 1, "qwen38-27b-mmproj-f16.gguf", "draft/qwen38-27b-mtp-Q8_0.gguf", "", "", ""}, + {"Q8_0", "", 0, "", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"philbert440/Qwen3.8-27B-Uncensored-Cyber-GGUF", + {{"Qwen3.8-27B-Uncensored-Cyber-Q4_K_M.gguf"}, + {"Qwen3.8-27B-Uncensored-Cyber-Q6_K.gguf"}, + {"Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf"}, + {"mmproj-Qwen3.8-27B-Uncensored-Cyber-BF16.gguf"}, + {"mmproj-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf"}, + {"mtp-Qwen3.8-27B-Uncensored-Cyber-BF16.gguf"}, + {"mtp-Qwen3.8-27B-Uncensored-Cyber-Q4_0.gguf"}, + {"mtp-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf"}}, + { + {"", "Qwen3.8-27B-Uncensored-Cyber-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf", "mtp-Qwen3.8-27B-Uncensored-Cyber-Q4_0.gguf", "", "", ""}, + {"Q4_K_M", "Qwen3.8-27B-Uncensored-Cyber-Q4_K_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf", "mtp-Qwen3.8-27B-Uncensored-Cyber-Q4_0.gguf", "", "", ""}, + {"Q8_0", "Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf", "mtp-Qwen3.8-27B-Uncensored-Cyber-Q8_0.gguf", "", "", ""}, + {"BF16", "", 0, "", "mtp-Qwen3.8-27B-Uncensored-Cyber-BF16.gguf", "", "", ""}}, + {"Q8_0", "Q6_K", "Q4_K_M"}}, + + {"ProCreations/grug-27b-qat-q4-gguf", + {{"grug-27b-qat-Q4_K_M.gguf"}, + {"mmproj-grug-27b-f16.gguf"}}, + { + {"", "grug-27b-qat-Q4_K_M.gguf", 1, "mmproj-grug-27b-f16.gguf", "", "", "", ""}, + {"Q4_K_M", "grug-27b-qat-Q4_K_M.gguf", 1, "mmproj-grug-27b-f16.gguf", "", "", "", ""}, + {"F16", "", 0, "", "", "", "", ""}}, + {"Q4_K_M"}}, + + {"QuantaPlanta/MiMo-V2.6-Flash-MOPD-MXFP4-GGUF", + {{"MiMo-V2.6-Flash-MOPD-MXFP4.gguf"}, + {"mmproj-MiMo-V2.6-Flash-MOPD-Q8_0.gguf"}, + {"mmproj-MiMo-V2.6-Flash-MOPD-bf16.gguf"}, + {"mmproj-MiMo-V2.6-Flash-MOPD-f16.gguf"}}, + { + {"", "MiMo-V2.6-Flash-MOPD-MXFP4.gguf", 1, "mmproj-MiMo-V2.6-Flash-MOPD-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "", 0, "", "", "", "", ""}, + {"MXFP4", "MiMo-V2.6-Flash-MOPD-MXFP4.gguf", 1, "mmproj-MiMo-V2.6-Flash-MOPD-Q8_0.gguf", "", "", "", ""}}, + {"MXFP4"}}, + + {"Qwen/Qwen3-VL-2B-Instruct-GGUF", + {{"Qwen3VL-2B-Instruct-F16.gguf"}, + {"Qwen3VL-2B-Instruct-Q4_K_M.gguf"}, + {"Qwen3VL-2B-Instruct-Q8_0.gguf"}, + {"mmproj-Qwen3VL-2B-Instruct-F16.gguf"}, + {"mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf"}}, + { + {"", "Qwen3VL-2B-Instruct-Q4_K_M.gguf", 1, "mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3VL-2B-Instruct-Q4_K_M.gguf", 1, "mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3VL-2B-Instruct-Q8_0.gguf", 1, "mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", "", "", "", ""}, + {"F16", "Qwen3VL-2B-Instruct-F16.gguf", 1, "mmproj-Qwen3VL-2B-Instruct-F16.gguf", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "F16"}}, + + {"ReadyArt/Serenity-26B-A4B-GGUF", + {{"Serenity-26B-A4B-HB16-Q6_K.gguf"}, + {"Serenity-26B-A4B-HB16-Q8_0.gguf"}, + {"Serenity-26B-A4B-IQ4_XS.gguf"}, + {"Serenity-26B-A4B-Q4_K_M.gguf"}, + {"Serenity-26B-A4B-Q5_K_M.gguf"}, + {"Serenity-26B-A4B-Q6_K.gguf"}, + {"Serenity-26B-A4B-Q8_0.gguf"}}, + { + {"", "Serenity-26B-A4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Serenity-26B-A4B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Serenity-26B-A4B-HB16-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q6_K", "Serenity-26B-A4B-HB16-Q6_K.gguf", 1, "", "", "", "", ""}, + {"Q5_K_M", "Serenity-26B-A4B-Q5_K_M.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "IQ4_XS"}}, + + {"realrebelai/KREA-2_GGUFs", + {{"BASE/Krea-2-Base-Q3_K_M.gguf"}, + {"BASE/Krea-2-Base-Q4_K_M.gguf"}, + {"BASE/Krea-2-Base-Q8_0.gguf"}, + {"TURBO/Krea-2-Turbo-Q4_K_M.gguf"}, + {"TURBO/Krea-2-Turbo-Q8_0.gguf"}}, + { + {"", "BASE/Krea-2-Base-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "BASE/Krea-2-Base-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "BASE/Krea-2-Base-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q3_K_M", "BASE/Krea-2-Base-Q3_K_M.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_M"}}, + + {"realrebelai/MiniMax-H3_GGUFs", + {{"MiniMax-H3-FL2VA-Q3_K_M.gguf"}, + {"MiniMax-H3-FL2VA-Q4_K_M.gguf"}, + {"MiniMax-H3-REF2VA-Q3_K_M.gguf"}, + {"MiniMax-H3-REF2VA-Q4_K_M.gguf"}, + {"qwen3vl-32B-MiniMax-H3-Q2_K.gguf"}, + {"qwen3vl-32B-MiniMax-H3-Q4_K_M.gguf"}}, + { + {"", "MiniMax-H3-FL2VA-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "MiniMax-H3-FL2VA-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q3_K_M", "MiniMax-H3-FL2VA-Q3_K_M.gguf", 1, "", "", "", "", ""}, + {"Q2_K", "qwen3vl-32B-MiniMax-H3-Q2_K.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M", "Q2_K", "Q3_K_M"}}, + + {"realrebelai/Wan-Animate-2_GGUFs", + {{"Distilled/Wan-Animate-2-14B-TURBO-Q2_K.gguf"}, + {"Distilled/Wan-Animate-2-14B-TURBO-Q4_K_M.gguf"}, + {"Distilled/Wan-Animate-2-14B-TURBO-Q4_K_S.gguf"}, + {"Distilled/Wan-Animate-2-14B-TURBO-Q8_0.gguf"}}, + { + {"", "Distilled/Wan-Animate-2-14B-TURBO-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Distilled/Wan-Animate-2-14B-TURBO-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Distilled/Wan-Animate-2-14B-TURBO-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q2_K", "Distilled/Wan-Animate-2-14B-TURBO-Q2_K.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_S", "Q4_K_M", "Q2_K"}}, + + {"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF", + {{"gemma-4-26B-A4B-it-qat-heretic-UD-Q4_K_XL.gguf"}, + {"gemma-4-26B-A4B-it-qat-heretic-f16.gguf"}, + {"mmproj-gemma-4-26B-A4B-it-qat-heretic-F16.gguf"}}, + { + {"", "gemma-4-26B-A4B-it-qat-heretic-UD-Q4_K_XL.gguf", 1, "mmproj-gemma-4-26B-A4B-it-qat-heretic-F16.gguf", "", "", "", ""}, + {"UD-Q4_K_XL", "gemma-4-26B-A4B-it-qat-heretic-UD-Q4_K_XL.gguf", 1, "mmproj-gemma-4-26B-A4B-it-qat-heretic-F16.gguf", "", "", "", ""}, + {"F16", "gemma-4-26B-A4B-it-qat-heretic-f16.gguf", 1, "mmproj-gemma-4-26B-A4B-it-qat-heretic-F16.gguf", "", "", "", ""}}, + {"F16", "Q4_K_XL"}}, + + {"SC117/Spark-X2.5-4B-abliterated-FIT-GGUF", + {{"Spark-X2.5-4B-abliterated-BF16.gguf"}, + {"Spark-X2.5-4B-abliterated-FIT-COMPACT-2.43GiB-Q4_K.gguf"}, + {"Spark-X2.5-4B-abliterated-FIT-QUALITY-3.15GiB-Q6_K.gguf"}}, + { + {"", "Spark-X2.5-4B-abliterated-BF16.gguf", 1, "", "", "", "", ""}, + {"BF16", "Spark-X2.5-4B-abliterated-BF16.gguf", 1, "", "", "", "", ""}, + {"Q6_K", "Spark-X2.5-4B-abliterated-FIT-QUALITY-3.15GiB-Q6_K.gguf", 1, "", "", "", "", ""}}, + {"Q6_K", "Q4_K", "BF16"}}, + + {"SC117/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-GGUF", + {{"IQ2_XS/Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS", 2}, + {"Q2_0/Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-Q2_0", 2}, + {"Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ3_S", 2}, + {"mmproj-Qwen3.8-Flash-Next-BF16.gguf"}, + {"mtp-Qwen3.8-Flash-Next-Q8_0.gguf"}, + {"mtp-q2_0.gguf"}}, + { + {"", "IQ2_XS/Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"Q8_0", "", 0, "", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"IQ2_XS", "IQ2_XS/Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ2_XS-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"IQ3_S", "Swift-Qwen3.8-Flash-Next-GSQ-RCO-abliterated-IQ3_S-00001-of-00002.gguf", 2, "mmproj-Qwen3.8-Flash-Next-BF16.gguf", "mtp-q2_0.gguf", "", "", ""}, + {"BF16", "", 0, "", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}}, + {"IQ3_S", "Q2_0", "IQ2_XS"}}, + + {"scragnog/MOSS-Music-8B-Instruct-GGUF", + {{"moss-aud-f16.gguf"}, + {"moss-lm-f16.gguf"}, + {"moss-lm-q8_0.gguf"}}, + { + {"", "moss-lm-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "moss-lm-q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "moss-aud-f16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "F16"}}, + + {"Snowflake/snowflake-arctic-embed-m-v1.5", + {{"gguf/snowflake-arctic-embed-m-v1.5-bf16.gguf"}, + {"gguf/snowflake-arctic-embed-m-v1.5-q8_0.gguf"}, + {"gguf/snowflake-arctic-embed-m-v1.5-tq2_0.gguf"}}, + { + {"", "gguf/snowflake-arctic-embed-m-v1.5-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "gguf/snowflake-arctic-embed-m-v1.5-q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "gguf/snowflake-arctic-embed-m-v1.5-bf16.gguf", 1, "", "", "", "", ""}, + {"TQ2_0", "gguf/snowflake-arctic-embed-m-v1.5-tq2_0.gguf", 1, "", "", "", "", ""}}, + {"TQ2_0", "Q8_0", "BF16"}}, + + {"Solstice-AI/GLM-5.3-Flash-UNCENSORED-mlx-oQ4e-DFlash2", + {{"mmproj-BF16.gguf"}, + {"speculative/GLM-5.3-Flash-DFlash2-bf16.gguf"}}, + { + {"", "speculative/GLM-5.3-Flash-DFlash2-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "speculative/GLM-5.3-Flash-DFlash2-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"BF16"}}, + + {"superwhisper/s1-mini-GGUF", + {{"s1-mini-f16.gguf"}, + {"s1-mini-q4_k_m.gguf"}}, + { + {"", "s1-mini-q4_k_m.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "s1-mini-q4_k_m.gguf", 1, "", "", "", "", ""}, + {"F16", "s1-mini-f16.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M", "F16"}}, + + {"tencent/Hy-MT2-7B-GGUF", + {{"HY-MT2-7B-Q6_K.gguf"}, + {"HY-MT2-7B-Q8_0.gguf"}, + {"Hy-MT2-7B-Q4_K_M.gguf"}}, + { + {"", "Hy-MT2-7B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Hy-MT2-7B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "HY-MT2-7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q6_K", "HY-MT2-7B-Q6_K.gguf", 1, "", "", "", "", ""}}, + {"Q4_K_M", "Q8_0", "Q6_K"}}, + + {"TrevorJS/gemma-4-26B-A4B-it-uncensored-GGUF", + {{"gemma-4-26B-A4B-it-uncensored-Q4_K_M.gguf"}, + {"gemma-4-26B-A4B-it-uncensored-Q8_0.gguf"}}, + { + {"", "gemma-4-26B-A4B-it-uncensored-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "gemma-4-26B-A4B-it-uncensored-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "gemma-4-26B-A4B-it-uncensored-Q8_0.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M"}}, + + {"ukisai/Swift-1.5-Qwen3.8-Flash-Next-GGUF", + {{"IQ1_M/Swift-1.5-Qwen3.8-Flash-Next-IQ1_M", 2}, + {"Q3_K_L/Swift-1.5-Qwen3.8-Flash-Next-Q3_K_L", 3}, + {"Q4_K_M/Swift-1.5-Qwen3.8-Flash-Next-Q4_K_M", 3}, + {"Q8_0/Swift-1.5-Qwen3.8-Flash-Next-Q8_0", 5}, + {"mmproj-Swift-1.5-Qwen3.8-Flash-Next-BF16.gguf"}}, + { + {"", "Q4_K_M/Swift-1.5-Qwen3.8-Flash-Next-Q4_K_M-00001-of-00003.gguf", 3, "mmproj-Swift-1.5-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Q4_K_M/Swift-1.5-Qwen3.8-Flash-Next-Q4_K_M-00001-of-00003.gguf", 3, "mmproj-Swift-1.5-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Swift-1.5-Qwen3.8-Flash-Next-Q8_0-00001-of-00005.gguf", 5, "mmproj-Swift-1.5-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"IQ1_M", "IQ1_M/Swift-1.5-Qwen3.8-Flash-Next-IQ1_M-00001-of-00002.gguf", 2, "mmproj-Swift-1.5-Qwen3.8-Flash-Next-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_L", "IQ1_M"}}, + + {"unsloth/DeepSeek-R1-Distill-Qwen-32B-GGUF", + {{"DeepSeek-R1-Distill-Qwen-32B-F16/DeepSeek-R1-Distill-Qwen-32B-F16", 2}, + {"DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf"}, + {"DeepSeek-R1-Distill-Qwen-32B-Q8_0.gguf"}}, + { + {"", "DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "DeepSeek-R1-Distill-Qwen-32B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"F16", "DeepSeek-R1-Distill-Qwen-32B-F16/DeepSeek-R1-Distill-Qwen-32B-F16-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "F16"}}, + + {"unsloth/DeepSeek-V4-Flash-0731-GGUF", + {{"UD-IQ1_M/DeepSeek-V4-Flash-0731-UD-IQ1_M", 3}, + {"UD-IQ4_NL/DeepSeek-V4-Flash-0731-UD-IQ4_NL", 4}, + {"UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL", 5}, + {"dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"}, + {"dspark/dspark-DeepSeek-V4-Flash-0731-BF16.gguf"}}, + { + {"", "UD-IQ1_M/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf", 3, "", "", "", "", "dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"}, + {"Q8_0", "", 0, "", "", "", "", "dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"}, + {"UD-IQ1_M", "UD-IQ1_M/DeepSeek-V4-Flash-0731-UD-IQ1_M-00001-of-00003.gguf", 3, "", "", "", "", "dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00001-of-00005.gguf", 5, "", "", "", "", "dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"}}, + {"Q8_K_XL", "IQ4_NL", "IQ1_M"}}, + + {"unsloth/DeepSeek-V4-Flash-GGUF", + {{"UD-IQ1_M/DeepSeek-V4-Flash-UD-IQ1_M", 3}, + {"UD-IQ4_NL/DeepSeek-V4-Flash-UD-IQ4_NL", 4}, + {"UD-Q8_K_XL/DeepSeek-V4-Flash-UD-Q8_K_XL", 5}}, + { + {"", "UD-IQ1_M/DeepSeek-V4-Flash-UD-IQ1_M-00001-of-00003.gguf", 3, "", "", "", "", ""}, + {"UD-IQ1_M", "UD-IQ1_M/DeepSeek-V4-Flash-UD-IQ1_M-00001-of-00003.gguf", 3, "", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/DeepSeek-V4-Flash-UD-Q8_K_XL-00001-of-00005.gguf", 5, "", "", "", "", ""}}, + {"Q8_K_XL", "IQ4_NL", "IQ1_M"}}, + + {"unsloth/DeepSeek-V4-Pro-0813-GGUF", + {{"UD-Q4_K_XL/DeepSeek-V4-Pro-0813-UD-Q4_K_XL", 20}, + {"UD-Q8_K_XL/DeepSeek-V4-Pro-0813-UD-Q8_K_XL", 20}}, + { + {"", "UD-Q4_K_XL/DeepSeek-V4-Pro-0813-UD-Q4_K_XL-00001-of-00020.gguf", 20, "", "", "", "", ""}, + {"UD-Q4_K_XL", "UD-Q4_K_XL/DeepSeek-V4-Pro-0813-UD-Q4_K_XL-00001-of-00020.gguf", 20, "", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/DeepSeek-V4-Pro-0813-UD-Q8_K_XL-00001-of-00020.gguf", 20, "", "", "", "", ""}}, + {"Q8_K_XL", "Q4_K_XL"}}, + + {"unsloth/embeddinggemma-2-GGUF", + {{"embeddinggemma-2-BF16.gguf"}, + {"embeddinggemma-2-Q8_0.gguf"}, + {"embeddinggemma-2-UD-Q4_K_XL.gguf"}, + {"embeddinggemma-2-UD-Q6_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-Q8_0.gguf"}}, + { + {"", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"Q8_0", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"BF16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q6_K_XL", "embeddinggemma-2-UD-Q6_K_XL.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, + {"F16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q6_K_XL", "Q4_K_XL", "Q8_0", "BF16"}}, + + {"unsloth/GLM-4.7-Flash-GGUF", + {{"BF16/GLM-4.7-Flash-BF16", 2}, + {"GLM-4.7-Flash-Q4_K_M.gguf"}, + {"GLM-4.7-Flash-Q6_K.gguf"}, + {"GLM-4.7-Flash-Q8_0.gguf"}, + {"GLM-4.7-Flash-UD-TQ1_0.gguf"}}, + { + {"", "GLM-4.7-Flash-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "GLM-4.7-Flash-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "GLM-4.7-Flash-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "BF16/GLM-4.7-Flash-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"UD-TQ1_0", "GLM-4.7-Flash-UD-TQ1_0.gguf", 1, "", "", "", "", ""}}, + {"TQ1_0", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"unsloth/GLM-5.3-Flash-GGUF", + {{"BF16/GLM-5.3-Flash-BF16", 14}, + {"Q8_0/GLM-5.3-Flash-Q8_0", 8}, + {"UD-IQ4_XS/GLM-5.3-Flash-UD-IQ4_XS", 5}, + {"UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL", 7}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}}, + { + {"", "Q8_0/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q6_K_XL", "UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf", 7, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q6_K_XL", "IQ4_XS", "Q8_0", "BF16"}}, + + {"unsloth/GLM-5.3-GGUF", + {{"BF16/GLM-5.3-BF16", 33}, + {"Q8_0/GLM-5.3-Q8_0", 17}, + {"UD-IQ4_XS/GLM-5.3-UD-IQ4_XS", 9}, + {"UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL", 16}}, + { + {"", "Q8_0/GLM-5.3-Q8_0-00001-of-00017.gguf", 17, "", "", "", "", ""}, + {"Q8_0", "Q8_0/GLM-5.3-Q8_0-00001-of-00017.gguf", 17, "", "", "", "", ""}, + {"BF16", "BF16/GLM-5.3-BF16-00001-of-00033.gguf", 33, "", "", "", "", ""}, + {"UD-Q6_K_XL", "UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00001-of-00016.gguf", 16, "", "", "", "", ""}}, + {"Q6_K_XL", "IQ4_XS", "Q8_0", "BF16"}}, + + {"unsloth/inkling-GGUF", + {{"BF16/inkling-GGUF-BF16", 41}, + {"Q8_0/inkling-GGUF-Q8_0", 19}, + {"UD-IQ1_S/inkling-UD-IQ1_S", 7}, + {"UD-Q4_K_XL/inkling-UD-Q4_K_XL", 14}, + {"mmproj-BF16.gguf"}}, + { + {"", "Q8_0/inkling-GGUF-Q8_0-00001-of-00019.gguf", 19, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/inkling-GGUF-Q8_0-00001-of-00019.gguf", 19, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/inkling-GGUF-BF16-00001-of-00041.gguf", 41, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q4_K_XL", "UD-Q4_K_XL/inkling-UD-Q4_K_XL-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q4_K_XL", "IQ1_S", "Q8_0", "BF16"}}, + + {"unsloth/Kimi-K2.6-GGUF", + {{"BF16/Kimi-K2.6-BF16", 46}, + {"UD-Q4_K_XL/Kimi-K2.6-UD-Q4_K_XL", 14}, + {"UD-Q8_K_XL/Kimi-K2.6-UD-Q8_K_XL", 14}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/Kimi-K2.6-UD-Q8_K_XL-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "Q4_K_XL", "BF16"}}, + + {"unsloth/Kimi-K2.7-Code-GGUF", + {{"UD-IQ1_M/Kimi-K2.7-Code-UD-IQ1_M", 8}, + {"UD-Q2_K_XL/Kimi-K2.7-Code-UD-Q2_K_XL", 8}, + {"UD-Q8_K_XL/Kimi-K2.7-Code-UD-Q8_K_XL", 14}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "UD-IQ1_M/Kimi-K2.7-Code-UD-IQ1_M-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-IQ1_M", "UD-IQ1_M/Kimi-K2.7-Code-UD-IQ1_M-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/Kimi-K2.7-Code-UD-Q8_K_XL-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q8_K_XL", "Q2_K_XL", "IQ1_M"}}, + + {"unsloth/Kimi-K3-GGUF", + {{"UD-IQ1_M/Kimi-K3-UD-IQ1_M", 15}, + {"UD-Q2_K_XL/Kimi-K3-UD-Q2_K_XL", 19}, + {"UD-TQ2_0/Kimi-K3-UD-TQ2_0", 13}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "UD-IQ1_M/Kimi-K3-UD-IQ1_M-00001-of-00015.gguf", 15, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-IQ1_M", "UD-IQ1_M/Kimi-K3-UD-IQ1_M-00001-of-00015.gguf", 15, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-TQ2_0", "UD-TQ2_0/Kimi-K3-UD-TQ2_0-00001-of-00013.gguf", 13, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"TQ2_0", "Q2_K_XL", "IQ1_M"}}, + + {"unsloth/LFM2.5-VL-3B-GGUF", + {{"LFM2.5-VL-3B-BF16.gguf"}, + {"LFM2.5-VL-3B-Q4_K_M.gguf"}, + {"LFM2.5-VL-3B-Q5_K_S.gguf"}, + {"LFM2.5-VL-3B-Q8_0.gguf"}, + {"LFM2.5-VL-3B-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "LFM2.5-VL-3B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "LFM2.5-VL-3B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "LFM2.5-VL-3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "LFM2.5-VL-3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "Q8_0", "Q5_K_S", "Q4_K_M", "BF16"}}, + + {"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", + {{"BF16/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", 2}, + {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf"}, + {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q3_K_XL.gguf"}, + {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q8_K_XL.gguf"}}, + { + {"", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "BF16/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"UD-Q8_K_XL", "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q8_K_XL.gguf", 1, "", "", "", "", ""}}, + {"Q8_K_XL", "Q3_K_XL", "Q8_0", "BF16"}}, + + {"unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF", + {{"BF16/Qwen3-30B-A3B-Instruct-2507-BF16", 2}, + {"Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf"}, + {"Qwen3-30B-A3B-Instruct-2507-Q6_K.gguf"}, + {"Qwen3-30B-A3B-Instruct-2507-Q8_0.gguf"}, + {"Qwen3-30B-A3B-Instruct-2507-UD-TQ1_0.gguf"}}, + { + {"", "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Qwen3-30B-A3B-Instruct-2507-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "BF16/Qwen3-30B-A3B-Instruct-2507-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, + {"UD-TQ1_0", "Qwen3-30B-A3B-Instruct-2507-UD-TQ1_0.gguf", 1, "", "", "", "", ""}}, + {"TQ1_0", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"unsloth/Qwen3-VL-30B-A3B-Instruct-GGUF", + {{"BF16/Qwen3-VL-30B-A3B-Instruct-BF16", 2}, + {"Qwen3-VL-30B-A3B-Instruct-Q4_K_M.gguf"}, + {"Qwen3-VL-30B-A3B-Instruct-Q6_K.gguf"}, + {"Qwen3-VL-30B-A3B-Instruct-Q8_0.gguf"}, + {"Qwen3-VL-30B-A3B-Instruct-UD-TQ1_0.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3-VL-30B-A3B-Instruct-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3-VL-30B-A3B-Instruct-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3-VL-30B-A3B-Instruct-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-TQ1_0", "Qwen3-VL-30B-A3B-Instruct-UD-TQ1_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"TQ1_0", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"unsloth/Qwen3.5-27B-GGUF", + {{"BF16/Qwen3.5-27B-BF16", 2}, + {"Qwen3.5-27B-Q4_K_M.gguf"}, + {"Qwen3.5-27B-Q6_K.gguf"}, + {"Qwen3.5-27B-Q8_0.gguf"}, + {"Qwen3.5-27B-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3.5-27B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-27B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-27B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Qwen3.5-27B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, + + {"unsloth/Qwen3.5-35B-A3B-GGUF", + {{"BF16/Qwen3.5-35B-A3B-BF16", 2}, + {"Qwen3.5-35B-A3B-Q4_K_M.gguf"}, + {"Qwen3.5-35B-A3B-Q8_0.gguf"}, + {"Qwen3.5-35B-A3B-UD-IQ3_S.gguf"}, + {"Qwen3.5-35B-A3B-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3.5-35B-A3B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Qwen3.5-35B-A3B-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.5-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Qwen3.5-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "IQ3_S", "Q8_0", "Q4_K_M", "BF16"}}, + + {"unsloth/Qwen3.5-397B-A17B-GGUF", + {{"BF16/Qwen3.5-397B-A17B-BF16", 17}, + {"Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M", 6}, + {"Q8_0/Qwen3.5-397B-A17B-Q8_0", 10}, + {"UD-IQ3_XXS/Qwen3.5-397B-A17B-UD-IQ3_XXS", 4}, + {"UD-Q8_K_XL/Qwen3.5-397B-A17B-UD-Q8_K_XL", 10}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4_K_M", "Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Qwen3.5-397B-A17B-Q8_0-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/Qwen3.5-397B-A17B-UD-Q8_K_XL-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "IQ3_XXS", "Q8_0", "Q4_K_M", "BF16"}}, + + {"unsloth/Qwen3.6-35B-A3B-MTP-GGUF", + {{"BF16/Qwen3.6-35B-A3B-BF16", 2}, + {"Qwen3.6-35B-A3B-Q8_0.gguf"}, + {"Qwen3.6-35B-A3B-UD-Q3_K_M.gguf"}, + {"Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf"}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Qwen3.6-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Qwen3.6-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, + + {"unsloth/Qwen3.8-Flash-Next-GGUF", + {{"BF16/Qwen3.8-Flash-Next-BF16", 8}, + {"MTP/mtp-Qwen3.8-Flash-Next-BF16.gguf"}, + {"MTP/mtp-Qwen3.8-Flash-Next-Q4_K_M.gguf"}, + {"MTP/mtp-Qwen3.8-Flash-Next-Q8_0.gguf"}, + {"MTP/mtp-Qwen3.8-Flash-Next-shared-BF16.gguf"}, + {"MTP/mtp-Qwen3.8-Flash-Next-shared-Q4_K_M.gguf"}, + {"MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"}, + {"Q8_0/Qwen3.8-Flash-Next-Q8_0", 6}, + {"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS", 3}, + {"UD-Q6_K_XL/Qwen3.8-Flash-Next-UD-Q6_K_XL", 6}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mtp-Qwen3.8-Flash-Next-Q8_0.gguf"}}, + { + {"", "Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"Q4_K_M", "", 0, "", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"Q8_0", "Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"BF16", "BF16/Qwen3.8-Flash-Next-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}, + {"UD-Q6_K_XL", "UD-Q6_K_XL/Qwen3.8-Flash-Next-UD-Q6_K_XL-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "", "", ""}}, + {"Q6_K_XL", "IQ4_XS", "Q8_0", "BF16"}}, + + {"unsloth/Step-3.7-Flash-GGUF", + {{"BF16/Step-3.7-Flash-BF16", 8}, + {"Q8_0/Step-3.7-Flash-Q8_0", 6}, + {"UD-Q3_K_M/Step-3.7-Flash-UD-Q3_K_M", 3}, + {"UD-Q8_K_XL/Step-3.7-Flash-UD-Q8_K_XL", 6}, + {"mmproj-BF16.gguf"}, + {"mmproj-F16.gguf"}, + {"mmproj-F32.gguf"}}, + { + {"", "Q8_0/Step-3.7-Flash-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q8_0", "Q8_0/Step-3.7-Flash-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, + {"UD-Q8_K_XL", "UD-Q8_K_XL/Step-3.7-Flash-UD-Q8_K_XL-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, + {"F16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}}, + {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, + + {"vmlinux/Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix-GGUF", + {{"Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix.gguf"}, + {"mmproj-BF16.gguf"}}, + { + {"", "Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"Q4MIX", "Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, + {"BF16", "", 0, "", "", "", "", ""}}, + {"Q4MIX"}}, + + {"WalkingCat/Soprano-1.1-80M-GGUF", + {{"Soprano-1.1-80M-GGUF/soprano-1.1-80m-bf16.gguf"}, + {"Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf"}}, + { + {"", "Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "Soprano-1.1-80M-GGUF/soprano-1.1-80m-bf16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "BF16"}}, + + {"yuxinlu1/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-GGUF", + {{"MTP/gemma-4-12B-it-MTP-BF16.gguf"}, + {"MTP/gemma-4-12B-it-MTP-Q8_0.gguf"}, + {"gemma4-v2-Q3_K_M.gguf"}, + {"gemma4-v2-Q4_K_M.gguf"}, + {"gemma4-v2-Q8_0.gguf"}}, + { + {"", "gemma4-v2-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "gemma4-v2-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "MTP/gemma-4-12B-it-MTP-Q8_0.gguf", 1, "", "", "", "", ""}, + {"BF16", "MTP/gemma-4-12B-it-MTP-BF16.gguf", 1, "", "", "", "", ""}}, + {"Q8_0", "Q4_K_M", "Q3_K_M", "BF16"}}, + + {"zetasepic/Qwen2.5-72B-Instruct-abliterated-GGUF", + {{"Qwen2.5-72B-Instruct-abliterated-Q3_K_M.gguf"}, + {"Qwen2.5-72B-Instruct-abliterated-Q4_K_M.gguf"}, + {"Qwen2.5-72B-Instruct-abliterated-Q6_K", 2}}, + { + {"", "Qwen2.5-72B-Instruct-abliterated-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q4_K_M", "Qwen2.5-72B-Instruct-abliterated-Q4_K_M.gguf", 1, "", "", "", "", ""}, + {"Q3_K_M", "Qwen2.5-72B-Instruct-abliterated-Q3_K_M.gguf", 1, "", "", "", "", ""}, + {"Q6_K", "Qwen2.5-72B-Instruct-abliterated-Q6_K-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"Q6_K", "Q4_K_M", "Q3_K_M"}}, +}; + +static std::vector real_files(const real_repo & r) { + std::vector out; + for (const auto & f : r.files) { + if (f.parts == 0) { + out.push_back(f.name); + continue; + } + for (int i = 1; i <= f.parts; ++i) { + char suffix[32]; + snprintf(suffix, sizeof(suffix), "-%05d-of-%05d.gguf", i, f.parts); + out.push_back(std::string(f.name) + suffix); + } + } + return out; +} + +// every mismatch is reported before the test fails, so one run lists them all +static int g_real_failures = 0; + +static void expect_eq(const std::string & what, const std::string & actual, const std::string & expected) { + if (actual != expected) { + fprintf(stderr, " [%s] %s\n actual: '%s'\n expected: '%s'\n", + g_context.c_str(), what.c_str(), actual.c_str(), expected.c_str()); + g_real_failures++; + } +} + +static void check_real_plan(const real_repo & r, const real_plan & p) { + g_context = std::string(r.id) + (p.tag[0] ? std::string(":") + p.tag : ""); + + common_download_opts opts; + opts.download_mmproj = true; + opts.download_mtp = true; + opts.download_dflash = true; + opts.download_eagle3 = true; + opts.download_dspark = true; + + auto plan = common_download_get_hf_plan(model_ref(g_context), opts); + + expect_eq("primary", plan.primary.path, p.primary); + expect_eq("parts", std::to_string(plan.model_files.size()), std::to_string(p.parts)); + expect_eq("mmproj", plan.mmproj.path, p.mmproj); + expect_eq("mtp", plan.mtp.path, p.mtp); + expect_eq("dflash", plan.dflash.path, p.dflash); + expect_eq("eagle3", plan.eagle3.path, p.eagle3); + expect_eq("dspark", plan.dspark.path, p.dspark); +} + +// the listing reads the cache, so the repo is materialized there as a snapshot; +// a path the filesystem refuses (MAX_PATH on Windows) leaves the listing unchecked +static void check_real_listing(const real_repo & r, const std::vector & files) { + g_context = std::string(r.id) + " listing"; + + std::string folder = r.id; + string_replace_all(folder, "/", "--"); + const std::filesystem::path repo_dir = cache_dir / ("models--" + folder); + std::filesystem::create_directories(repo_dir / "refs"); + { std::ofstream(repo_dir / "refs" / "main") << COMMIT; } + for (const auto & f : files) { + std::filesystem::path p = cached(r.id, f); + std::error_code ec; + std::filesystem::create_directories(p.parent_path(), ec); + { std::ofstream(p) << "gguf"; } + if (!std::filesystem::exists(p, ec)) { + printf(" %s: listing unchecked, the filesystem refuses %s\n", r.id, f.c_str()); + std::filesystem::remove_all(repo_dir, ec); + return; + } + } + + std::vector actual; + for (const auto & e : common_list_cached_models()) { + if (e.repo == r.id) { + actual.push_back(e.tag); + } + } + std::sort(actual.begin(), actual.end()); + auto expected = r.listing; + std::sort(expected.begin(), expected.end()); + expect_eq("tags", string_join(actual, " "), string_join(expected, " ")); + + std::filesystem::remove_all(repo_dir); +} + +static void test_real_repos() { + printf("test-model-resolution: real listings on %zu repos\n", sizeof(real_repos) / sizeof(real_repos[0])); + + for (const auto & r : real_repos) { + auto files = real_files(r); + g_repos.clear(); + g_repos[r.id] = files; + for (const auto & p : r.plans) { + check_real_plan(r, p); + } + check_real_listing(r, files); + } + g_repos.clear(); + + if (g_real_failures > 0) { + fprintf(stderr, "test-model-resolution: %d mismatches on real listings\n", g_real_failures); + std::abort(); + } +} + // // end-to-end assembly: real CLI parsing, real handler init resolving over the // loopback, downloads skipped by flipping offline before apply @@ -631,6 +2880,7 @@ int main(void) { test_plan_partials(); test_sidecar_removal(); test_shared_removal(); + test_real_repos(); test_task_assembly(); server.stop(); diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 1d7d65ef86..faabd48406 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -75,6 +75,7 @@ static void inject_test_response_marker() { struct mtmd_cli_context { mtmd::context_ptr ctx_vision; + mtmd_helper_cap mtmd_cap = {}; common_init_result_ptr llama_init; llama_model * model; @@ -120,7 +121,8 @@ struct mtmd_cli_context { init_vision_context(params); - if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) { + mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get()); + if (!mtmd_cap.chat) { LOG_ERR("Model does not support chat mode\n"); LOG_ERR("Hint: for TTS models, please use llama-tts\n"); exit(1); @@ -565,7 +567,7 @@ int main(int argc, char ** argv) { if (g_is_interrupted) return 130; - if (mtmd_support_audio_stream(ctx.ctx_vision.get())) { + if (ctx.mtmd_cap.audio_stream) { if (params.image.size() != 1) { LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n"); return 1; @@ -624,7 +626,7 @@ int main(int argc, char ** argv) { if (mtmd_support_audio(ctx.ctx_vision.get())) { LOG("\n /audio load an audio"); } - if (mtmd_helper_support_video(ctx.ctx_vision.get())) { + if (ctx.mtmd_cap.video) { LOG("\n /video load a video"); } LOG("\n /clear clear the chat history"); diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index eba034f79e..d06af8f026 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -501,15 +501,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt); } -bool mtmd_helper_support_video(const mtmd_context * ctx) { -#ifdef MTMD_VIDEO - return mtmd_support_vision(ctx); -#else - GGML_UNUSED(ctx); - return false; -#endif -} - // // Video input helpers // @@ -1033,17 +1024,40 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, #endif } -bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) { +mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mtmd_context * mctx) { + mtmd_helper_cap cap = {}; + cap.chat = true; + cap.text_gen = true; if (!mctx) { - return true; + return cap; } - auto * model = llama_get_model(lctx); - auto * tmpl = llama_model_chat_template(model, nullptr); - auto info = mtmd_gen_audio_get_info(mctx); + cap.vision = mtmd_support_vision(mctx); + cap.audio = mtmd_support_audio(mctx); + cap.audio_stream = mtmd_support_audio_stream(mctx); +#ifdef MTMD_VIDEO + cap.video = cap.vision; +#endif + const auto gen_audio_type = mtmd_gen_audio_get_info(mctx).type; + cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE; + cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS + || gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS; // tts-only model cannot be used for chat (no chat template) - bool is_tts_only = info.type != MTMD_GEN_AUDIO_TYPE_NONE && tmpl == nullptr; + cap.chat = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr; - return !is_tts_only; + return cap; +} + +bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) { + return mtmd_helper_get_capabilities(llama_get_model(lctx), mctx).chat; +} + +bool mtmd_helper_support_video(const mtmd_context * ctx) { +#ifdef MTMD_VIDEO + return mtmd_support_vision(ctx); +#else + GGML_UNUSED(ctx); + return false; +#endif } diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index acbfefcbae..e46b5d36a4 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -45,9 +45,6 @@ MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void); // Note: this also call mtmd_log_set() internally MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); -// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). -MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx); - struct mtmd_helper_bitmap_wrapper { mtmd_bitmap * bitmap; mtmd_helper_video * video_ctx; @@ -186,8 +183,31 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, mtmd_bitmap ** out_bitmap, char ** out_text); -// return true if model can be used for chat -MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx); +struct mtmd_helper_cap { + bool chat; // false for tts-only model (no chat template) + + // input capabilities + bool vision; + bool audio; + bool video; // also depending on MTMD_VIDEO build option + + // realtime capabilities + bool audio_stream; // realtime transcription, see mtmd_audio_stream_init() + + // output capabilities + bool text_gen; // always true for now + bool audio_gen; + bool audio_gen_spk_ref; // accepts a speaker reference audio for voice cloning +}; +typedef struct mtmd_helper_cap mtmd_helper_cap; + +// mctx can be NULL (text-only model) +MTMD_API struct mtmd_helper_cap mtmd_helper_get_capabilities(const struct llama_model * model, const struct mtmd_context * mctx); + +DEPRECATED(MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx), + "use mtmd_helper_get_capabilities() instead"); +DEPRECATED(MTMD_API bool mtmd_helper_support_video(const struct mtmd_context * ctx), + "use mtmd_helper_get_capabilities() instead"); // // Audio generation helpers diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a5cba1d71d..9043fa2c02 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1012,16 +1012,17 @@ public: llama_model * model_tgt = nullptr; mtmd_context * mctx = nullptr; + mtmd_helper_cap mtmd_cap = {}; // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default(); const llama_vocab * vocab = nullptr; bool has_cap_tts() const { - return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE; + return mtmd_cap.audio_gen; } bool has_cap_chat() const { - return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx); + return mtmd_cap.chat; } server_decision_context decision; @@ -1397,6 +1398,8 @@ private: } } + mtmd_cap = mtmd_helper_get_capabilities(model_tgt, mctx); + if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) { if (params_base.ctx_shift) { params_base.ctx_shift = false; @@ -1705,9 +1708,9 @@ private: /* reasoning_format */ params_base.reasoning_format, /* chat_template_kwargs */ params_base.default_template_kwargs, /* tmpls */ std::move(chat_templates), - /* allow_image */ mctx ? mtmd_support_vision(mctx) : false, - /* allow_audio */ mctx ? mtmd_support_audio (mctx) : false, - /* allow_video */ mctx ? mtmd_helper_support_video(mctx) : false, + /* allow_image */ mtmd_cap.vision, + /* allow_audio */ mtmd_cap.audio, + /* allow_video */ mtmd_cap.video, /* enable_thinking */ enable_thinking, /* reasoning_budget */ params_base.sampling.reasoning_budget_tokens, /* reasoning_budget_msg */ params_base.sampling.reasoning_budget_message, @@ -5127,7 +5130,7 @@ server_context_meta server_context::get_meta() const { /* has_inp_video */ impl->chat_params.allow_video, /* has_cap_chat */ impl->has_cap_chat(), /* has_cap_tts */ impl->has_cap_tts(), - /* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx), + /* has_audio_stream */ impl->mtmd_cap.audio_stream, /* json_ui_settings */ impl->json_ui_settings, /* slot_n_ctx */ impl->n_ctx_slot(), /* pooling_type */ llama_pooling_type(impl->ctx_tgt), diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index e1e006f101..5751401328 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -533,7 +533,7 @@ void server_model_meta::update_caps(const common_params & base) { std::vector output_modalities = {"text"}; if (!params.model.path.empty()) { const common_gguf_info info = common_get_gguf_info(params.model.path); - // same rule as mtmd_helper_model_can_chat() + // same rule as mtmd_helper_get_capabilities() const bool can_chat = !gen_audio || info.has_chat_tmpl; output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio); n_ctx_train = info.n_ctx_train; diff --git a/tools/ui/README.md b/tools/ui/README.md index 2ecbc8f512..c6ec54721c 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -406,8 +406,7 @@ Hooks are the thin view-layer between components and stores: they own UI concern | `use-chat-message-edit-context` | Message edit context (content + extras) | | `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | | `use-chat-screen-file-upload` | File upload queue + capability validation | -| `use-chat-screen-scroll` | Scroll container binding + navigation guard | -| `use-auto-scroll` | Auto-scroll controller for streaming | +| `use-stick-to-bottom` | Follows new content while the reader sits at the bottom of a scroller | | `use-marquee-selection` | Shift+click / marquee range selection | | `use-keyboard-shortcuts` | Global keyboard shortcuts | | `use-settings-navigation` | Settings section navigation | diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 91edac1d8b..639577134f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -144,6 +144,7 @@ ); const pickers = useChatFormPickers({ + canCompact: () => chatStore.canCompact(conversationsStore.activeMessages), focusInput: refocusInput, getCaretOffset: () => inputRef?.getCaretOffset(), getCwd: () => cwd, @@ -163,6 +164,7 @@ .getServers() .some((s) => s.enabled && prefs.isServerToolsEnabled(s.id) && s.url.trim()); }, + onCompact: () => chatStore.compactConversation(), openModelSelector: () => chatFormActionsRef?.openModelSelector(), setCaretOffset: (offset) => inputRef?.setCaretOffset(offset), setValue: (v) => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte index 29de5ec263..e468c594c0 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte @@ -30,9 +30,9 @@ {#snippet triggerLabel()} {#if reasoning.isOff} - + {:else} - + {/if} {#if !isDefault} @@ -102,9 +102,9 @@ + {/if} +{/snippet} + {#snippet body()} {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded} @@ -145,6 +188,8 @@ {@render body()} + + {@render compactAction()} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 46d05338b6..0c32882ef6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -2,6 +2,7 @@ import { goto } from '$app/navigation'; import { ChatMessageAssistant, + ChatMessageCompaction, ChatMessageMcpPrompt, ChatMessageSynthetic, ChatMessageSystem, @@ -14,7 +15,7 @@ SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants'; import { setChatMessageActionsContext, setChatMessageEditContext } from '$lib/contexts'; - import { AgenticSectionType, AttachmentType, MessageRole } from '$lib/enums'; + import { AgenticSectionType, AttachmentType, MessageRole, MessageType } from '$lib/enums'; import { DatabaseService } from '$lib/services/database.service'; import { chatStore, conversationsStore, deviceStore } from '$lib/stores'; import type { @@ -60,6 +61,7 @@ // Synthetic cwd-change messages render with the folder-row UI instead // of a user bubble. The persisted flag is the single source of truth. let isSynthetic = $derived(Boolean(message.isSynthetic)); + let isCompaction = $derived(message.type === MessageType.COMPACTION); let rawEditContent = $derived.by(() => { if (message.role !== MessageRole.ASSISTANT) return undefined; @@ -313,6 +315,10 @@ chatActions.regenerateWithBranching(message, modelOverride); } + function handleRegenerateCompaction() { + chatActions.regenerateCompaction(message); + } + function handleContinue() { chatActions.continueAssistantMessage(message); } @@ -409,6 +415,8 @@ {:else if mcpPromptExtra} + {:else if isCompaction} + {:else if isSynthetic} {:else if message.role === MessageRole.USER} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index 15683c3c32..ff23acf3cf 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -52,6 +52,9 @@ let isStreaming = $derived(chatStore.isStreaming()); let hasNoContent = $derived(!message?.content?.trim()); let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming); + // A compaction summary reports its own progress on the compaction row, so + // this turn stays quiet while it runs instead of duplicating the indicator. + let isCompacting = $derived(chatStore.isCompacting(message.convId)); // during a router auto-load the message has no model yet: target the model frozen in the // persisted stream state (survives a reload), then fall back to the dropdown selection @@ -66,6 +69,7 @@ let showProcessingInfoTop = $derived( message?.role === MessageRole.ASSISTANT && isActivelyProcessing && + !isCompacting && hasNoContent && !isAgentic && isLastAssistantMessage @@ -74,6 +78,7 @@ let showProcessingInfoBottom = $derived( message?.role === MessageRole.ASSISTANT && isActivelyProcessing && + !isCompacting && (!hasNoContent || isAgentic) && isLastAssistantMessage ); @@ -212,7 +217,6 @@ onRawOutputToggle={(enabled) => (showRawOutput = enabled)} {onRegenerate} rawOutputEnabled={showRawOutput} - role={MessageRole.ASSISTANT} showRawOutputSwitch={currentConfig.showRawOutputSwitch} /> {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte index 0026c31c3b..bd07fd6dd7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte @@ -31,6 +31,15 @@ promptMs={liveLlm.prompt_ms} promptTokens={liveLlm.prompt_n} /> +{:else if showMessageStats && message.timings?.compaction} + {@const summary = message.timings.compaction} + {:else if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms} {@const agentic = message.timings.agentic} + import { FoldVertical, Loader2 } from '@lucide/svelte'; + import { + ChatMessageActionIcons, + ChatMessageAssistantStatistics, + ChatMessageEditForm, + CollapsibleContentBlock, + MarkdownContent + } from '$lib/components/app'; + import { COLLAPSIBLE_TRANSITION_MS } from '$lib/constants'; + import { getChatMessageEditContext } from '$lib/contexts'; + import { useProcessingState } from '$lib/hooks/use-processing-state.svelte'; + import { chatStore, conversationsStore, settingsStore } from '$lib/stores'; + import type { DatabaseMessage } from '$lib/types'; + import { compactionTokensBefore, formatNumber } from '$lib/utils'; + + interface Props { + class?: string; + message: DatabaseMessage; + onRegenerate: () => void; + } + + let { class: className = '', message, onRegenerate }: Props = $props(); + + const COMPACTION_HEADER = 'Conversation compacted'; + const COMPACTION_RUNNING_LABEL = 'Compacting conversation'; + const COMPACTION_GENERATING_LABEL = 'Generating summary'; + + // The shared row context, so the summary is editable and branchable through + // the same actions as a chat turn. + const editCtx = getChatMessageEditContext(); + + const currentConfig = $derived(settingsStore.config); + const hasContent = $derived(message.content.trim().length > 0); + + // Only the node a running summary writes to reports progress: a finished + // compaction further up the branch keeps its headline while another one runs. + const isSummarizing = $derived(chatStore.getCompactingNodeId(message.convId) === message.id); + + const processingState = useProcessingState(); + + // The summary request feeds the shared processing state, so the body shows + // the same prompt progress line a chat turn does while it waits. + $effect(() => { + if (isSummarizing) processingState.startMonitoring(); + }); + + // The header carries the run: the prompt progress until the summary starts + // streaming, the phase after that. There is nothing to collapse before the + // first summary token arrives. + const runningLabel = $derived.by(() => { + // The prompt is through: whatever streams in now is the summary itself. + if (hasContent) return `${COMPACTION_RUNNING_LABEL} — ${COMPACTION_GENERATING_LABEL}`; + + const progress = processingState.getPromptProgressText(); + + return progress + ? `${COMPACTION_RUNNING_LABEL} — ${progress}` + : `${COMPACTION_RUNNING_LABEL}...`; + }); + + // Shrinking the branch is the feature's own headline, so the finished + // toggle reports the context size it compacted. + const tokensBefore = $derived(compactionTokensBefore(conversationsStore.activeMessages, message)); + + const completedLabel = $derived( + tokensBefore === null + ? COMPACTION_HEADER + : `${COMPACTION_HEADER} from ${formatNumber(tokensBefore)}` + ); + + const title = $derived(isSummarizing || !hasContent ? runningLabel : completedLabel); + + let open = $state(false); + // A toggle by hand outranks the in-progress expansion, the way the agentic + // sections treat their own overrides. + let userChoseOpen = $state(null); + + // The progress lives in the body (as it does for an assistant turn), so the + // row opens while the summary runs and collapses once it is done. + $effect(() => { + if (userChoseOpen !== null) return; + + open = isSummarizing; + }); + + let rootEl = $state(); + // The scroll waiting for the body to reach its height, so closing the block + // before that leaves no scroll behind it. + let settleTimer: ReturnType | null = null; + + /** + * A toggle hands the reader the block's own boundary, just below the tabs: one + * scroll, and only once the body has its height - before that the page is too + * short to scroll the divider up there and the call would clamp. + */ + function handleToggle() { + userChoseOpen = open; + cancelSettledScroll(); + + if (!rootEl) return; + + settleTimer = setTimeout(() => { + settleTimer = null; + rootEl?.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }, COLLAPSIBLE_TRANSITION_MS); + } + + function cancelSettledScroll() { + if (settleTimer === null) return; + + clearTimeout(settleTimer); + settleTimer = null; + } + + $effect(() => cancelSettledScroll); + + +
+ {#if editCtx.isEditing} + + {:else} + + + + +
+ +
+ + {#if hasContent && !isSummarizing} + + + {/if} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte index 1163c8a94d..68ea17a37c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte @@ -5,7 +5,7 @@ ChatMessageMcpPromptContent } from '$lib/components/app'; import { getChatMessageEditContext } from '$lib/contexts'; - import { McpPromptVariant, MessageRole } from '$lib/enums'; + import { McpPromptVariant } from '$lib/enums'; import type { DatabaseMessageExtraMcpPrompt } from '$lib/types'; interface Props { @@ -36,7 +36,7 @@ {#if message.timestamp}
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte index 7f4db944fd..98bfa2e112 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte @@ -5,7 +5,7 @@ import { Card } from '$lib/components/ui/card'; import { INPUT_CLASSES } from '$lib/constants'; import { getChatMessageEditContext } from '$lib/contexts'; - import { KeyboardKey, MessageRole } from '$lib/enums'; + import { KeyboardKey } from '$lib/enums'; import { settingsStore } from '$lib/stores'; import { autoResizeTextarea, isIMEComposing } from '$lib/utils'; @@ -191,7 +191,7 @@ {#if message.timestamp}
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte index 8be39f892c..c2baf67a88 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte @@ -6,7 +6,7 @@ ChatMessageUserBubble } from '$lib/components/app/chat'; import { getChatMessageEditContext } from '$lib/contexts'; - import { ChatMessageStatisticsMode, MessageRole } from '$lib/enums'; + import { ChatMessageStatisticsMode } from '$lib/enums'; import { useProcessingState } from '$lib/hooks/use-processing-state.svelte'; import { chatStore, settingsStore } from '$lib/stores'; @@ -29,7 +29,11 @@ const processingState = useProcessingState(); const currentConfig = $derived(settingsStore.config); - const isActivelyProcessing = $derived(isLastUserMessage && chatStore.isLoading); + // A compaction summary reports its own progress on the compaction row, so + // the summarized turn does not show live reading stats while it runs. + const isActivelyProcessing = $derived( + isLastUserMessage && chatStore.isLoading && !chatStore.isCompacting(message.convId) + ); // For agentic turns, prefer the cumulative agentic.llm totals over per-call timings. let storedReadingStats = $derived.by(() => { @@ -107,7 +111,7 @@ {#if message.timestamp}
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionIcons/ChatMessageActionIcons.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionIcons/ChatMessageActionIcons.svelte index 68ec5c8468..041841d34a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionIcons/ChatMessageActionIcons.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionIcons/ChatMessageActionIcons.svelte @@ -10,11 +10,9 @@ import Label from '$lib/components/ui/label/label.svelte'; import { Switch } from '$lib/components/ui/switch'; import { getChatMessageActionsContext, getChatMessageEditContext } from '$lib/contexts'; - import { MessageRole } from '$lib/enums'; import { conversationsStore } from '$lib/stores'; interface Props { - role: MessageRole.USER | MessageRole.ASSISTANT; justify: 'start' | 'end'; actionsPosition: 'left' | 'right'; onRegenerate?: () => void; @@ -31,7 +29,6 @@ onRawOutputToggle, onRegenerate, rawOutputEnabled = false, - role, showRawOutputSwitch = false }: Props = $props(); @@ -81,11 +78,11 @@ - {#if role === MessageRole.ASSISTANT && onRegenerate} + {#if onRegenerate} onRegenerate()} tooltip="Regenerate" /> {/if} - {#if role === MessageRole.ASSISTANT && onContinue} + {#if onContinue} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte index e7acb9cc6f..25ea0b2bdf 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte @@ -14,11 +14,10 @@ interface Props { messages?: DatabaseMessage[]; - onUserAction?: () => void; onMessagesReady?: (messageCount: number) => void; } - let { messages = [], onMessagesReady, onUserAction }: Props = $props(); + let { messages = [], onMessagesReady }: Props = $props(); let allConversationMessages = $state([]); @@ -26,7 +25,6 @@ const chatActions: ChatMessageActions = { continueAssistantMessage: async (message: DatabaseMessage) => { - onUserAction?.(); await chatStore.continueAssistantMessage(message.id); refreshAllMessages(); }, @@ -52,7 +50,6 @@ newContent: string, newExtras?: DatabaseMessageExtra[] ) => { - onUserAction?.(); // in-place edit: the store already updated activeMessages and no // branch is created, so sibling info stays valid without a refetch await chatStore.editUserMessagePreserveResponses(message.id, newContent, newExtras); @@ -63,7 +60,6 @@ newContent: string, newExtras?: DatabaseMessageExtra[] ) => { - onUserAction?.(); await chatStore.editMessageWithBranching(message.id, newContent, newExtras); refreshAllMessages(); }, @@ -73,7 +69,6 @@ newContent: string, shouldBranch: boolean ) => { - onUserAction?.(); await chatStore.editAssistantMessage(message.id, newContent, shouldBranch); // only a branch changes sibling info; an in-place edit already @@ -92,8 +87,12 @@ await conversationsStore.navigateToSibling(siblingId); }, + regenerateCompaction: async (message: DatabaseMessage) => { + await chatStore.regenerateCompaction(message.id); + refreshAllMessages(); + }, + regenerateWithBranching: async (message: DatabaseMessage, modelOverride?: string) => { - onUserAction?.(); await chatStore.regenerateMessageWithBranching(message.id, modelOverride); refreshAllMessages(); } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte index 4bcc66da36..a502df7b47 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte @@ -1,5 +1,6 @@ {#if dragAndDrop.isDragOver} {/if} - { - scroll.handleScroll(e); - handleMobileScroll(); - - if (e.isTrusted && Date.now() > mobileScrollDownHintLockedUntil) { - mobileScrollDownHint = false; - } - }} -/> + stick.handleScroll()} /> {#if isServerLoading} @@ -332,9 +232,6 @@ { - handleSendLikeScroll(); - }} /> {/if} @@ -367,16 +264,8 @@ {/if}
- {#if (deviceStore.isMobile ? mobileScrollDownHint || isMobileUserScrolledUp : autoScroll.userScrolledUp) && page.url.hash.includes(ROUTES.CHAT) && page.params.id} - { - mobileScrollDownHint = false; - scroll.chatScrollContainer?.scrollTo({ - behavior: 'smooth', - top: scroll.chatScrollContainer.scrollHeight - }); - }} - /> + {#if !stick.atBottom && page.url.hash.includes(ROUTES.CHAT) && page.params.id} + stick.scrollToBottom('smooth')} /> {/if}
diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenDialogsAndAlerts.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenDialogsAndAlerts.svelte index feba9ee35d..d8ff0d2eee 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenDialogsAndAlerts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenDialogsAndAlerts.svelte @@ -1,8 +1,7 @@ +{#snippet toggleIcon(layoutClass: string)} + {#if iconUrl} + + {:else if IconComponent} + + {/if} +{/snippet} + +{#snippet dividerRow()} + + +
+ {@render toggleIcon('')} + + {@render toggleTitle('text-xs whitespace-nowrap')} + + {#if collapsible} + + {/if} +
+ + +{/snippet} + +{#snippet toggleTitle(layoutClass: string)} + + {#if titleSnippet} + {@render titleSnippet()} + {:else} + {title} + {/if} + + + {#if subtitle} + {subtitle} + {/if} +{/snippet} + { @@ -47,62 +120,60 @@ }} {open} > - -
- {#if iconUrl} - - {:else if IconComponent} - - {/if} - - - {#if titleSnippet} - {@render titleSnippet()} - {:else} - {title} - {/if} - - - {#if subtitle} - {subtitle} - {/if} + {#if variant === 'divider' && !collapsible} +
+ {@render dividerRow()}
+ {:else if variant === 'divider'} + + {@render dividerRow()} - Toggle content + + {:else} + + > +
+ {@render toggleIcon('mt-0.75')} - Toggle content - - - - - {#if open} -
-
- {@render children()} -
+ {@render toggleTitle('overflow-x-auto text-sm')}
- {/if} - + + + + Toggle content + + {/if} + + {#if collapsible} + + + +
+
+ {@render children()} +
+
+ + + {/if} diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index 895e9a80a7..4d5d7c2b9e 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -32,7 +32,7 @@ UI_DATA_ATTRS } from '$lib/constants'; import { BooleanString, ColorMode, UrlProtocol } from '$lib/enums'; - import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte'; + import { createStickToBottom } from '$lib/hooks/use-stick-to-bottom.svelte'; import { settingsStore } from '$lib/stores'; import type { DatabaseMessageExtra } from '$lib/types/database'; import { @@ -123,8 +123,8 @@ let streamingCodeScrollContainer = $state(); - // Auto-scroll controller for streaming code block content - const streamingAutoScroll = createAutoScrollController(); + // Follows the code that streams into the fixed-height block below. + const streamingScroll = createStickToBottom(); let pendingMarkdown: string | null = null; let isProcessing = false; @@ -800,18 +800,16 @@ } }); - // Auto-scroll for streaming code block + // Follow the streaming code block while the reader stays at its bottom; the + // resize observer does the pinning, so no polling for a stable height. $effect(() => { - streamingAutoScroll.setContainer(streamingCodeScrollContainer); - }); + streamingScroll.attach(streamingCodeScrollContainer); - $effect(() => { - streamingAutoScroll.updateInterval(incompleteCodeBlock !== null); + return () => streamingScroll.detach(); }); onDestroy(() => { cleanupEventListeners(); - streamingAutoScroll.destroy(); }); @@ -905,7 +903,7 @@
streamingAutoScroll.handleScroll()} + onscroll={() => streamingScroll.handleScroll()} > 

+	import { Trash2 } from '@lucide/svelte';
+	import { DialogConfirmation } from '$lib/components/app';
+
+	interface Props {
+		open: boolean;
+		onConfirm: () => void;
+		onCancel: () => void;
+	}
+
+	let { onCancel, onConfirm, open = $bindable() }: Props = $props();
+
+
+
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte
deleted file mode 100644
index 7780b67d26..0000000000
--- a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte
+++ /dev/null
@@ -1,85 +0,0 @@
-
-
- {
-		if (!open) uiStore.modelInformation = null;
-	}}
-	open={uiStore.modelInformation !== null}
->
-	 event.preventDefault()}
-		overlayClass="z-[70]"
-	>
-		
-			
-				
-
-				{#if uiStore.modelInformation}
-					{uiStore.modelInformation.model}
-				{/if}
-			
-
-			
-				Details of the model, from the server or the Hub.
-			
-		
-
-		

-			{#if uiStore.modelInformation}
-				
-			{/if}
-		

-	
-
diff --git a/tools/ui/src/lib/components/app/dialogs/index.ts b/tools/ui/src/lib/components/app/dialogs/index.ts
index 5c6a56f00e..f038c447b7 100644
--- a/tools/ui/src/lib/components/app/dialogs/index.ts
+++ b/tools/ui/src/lib/components/app/dialogs/index.ts
@@ -119,6 +119,23 @@ export { default as DialogConfirmation } from './DialogConfirmation.svelte';
  */
 export { default as DialogConfirmDownload } from './DialogConfirmDownload.svelte';
 
+/**
+ * **DialogConversationDelete** - Confirm deleting a conversation
+ *
+ * Destructive confirmation shared by every conversation delete entry point
+ * (keyboard shortcut, sidebar, mobile search).
+ *
+ * @example
+ * ```svelte
+ *  (showDelete = false)}
+ * />
+ * ```
+ */
+export { default as DialogConversationDelete } from './DialogConversationDelete.svelte';
+
 /**
  * **DialogConversationRename** - Rename a conversation
  *
@@ -403,17 +420,6 @@ export { default as DialogModelNotAvailable } from './DialogModelNotAvailable.sv
  */
 export { default as DialogConversationSelection } from './DialogConversationSelection.svelte';
 
-/**
- *
- * MODEL INFORMATION DIALOGS
- *
- * Dialogs for displaying model and server information.
- *
- */
-
-/** **DialogModelInformation** - model details a phone opens instead of the manager pane. */
-export { default as DialogModelInformation } from './DialogModelInformation.svelte';
-
 /**
  * **DialogMcpResourcesBrowser** - MCP resources browser dialog
  *
diff --git a/tools/ui/src/lib/components/app/misc/CollapsibleRegion.svelte b/tools/ui/src/lib/components/app/misc/CollapsibleRegion.svelte
index edbb5f7743..c1b3f7a890 100644
--- a/tools/ui/src/lib/components/app/misc/CollapsibleRegion.svelte
+++ b/tools/ui/src/lib/components/app/misc/CollapsibleRegion.svelte
@@ -1,4 +1,5 @@
 
 
 
@@ -325,69 +198,9 @@
 			bind:modalities={modalityFilter}
 			{groups}
 			{isFavorite}
-			onSelect={(option) => (selectedId = option.id)}
+			onSelect={(option) => uiStore.openModelDetails(option.id)}
 			{selectedId}
 			{toolbarEnd}
 		/>
- - - { - if (!open) selectedId = null; - }} - open={selected !== null} - shouldScaleBackground={false} -> - - Model details - - - Configuration, actions and metadata of the selected model. - - -
- {#if shownOption} - {#key shownId} - (selectedId = null)} - onToggleLoad={() => void toggleLoad(shownOption)} - onUseInChat={() => void useInChat(shownOption)} - onUseInNewChat={() => void useInNewChat(shownOption)} - option={shownOption} - /> - {/key} - {/if} -
- - - - diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelDrawer.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelDrawer.svelte new file mode 100644 index 0000000000..d9e7643aea --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelDrawer.svelte @@ -0,0 +1,188 @@ + + + + selected !== null, + (open) => { + if (!open) uiStore.closeModelDetails(); + } + } + direction={deviceStore.isMobile ? 'bottom' : 'right'} + shouldScaleBackground={false} +> + + Model details + + + Configuration, actions and metadata of the selected model. + + +
+ {#if shownOption} + {#key shownId} + uiStore.closeModelDetails()} + onToggleLoad={() => void toggleLoad(shownOption)} + onUseInChat={() => void useInChat(shownOption)} + onUseInNewChat={() => void useInNewChat(shownOption)} + option={shownOption} + /> + {/key} + {/if} +
+ + + + diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts index ccbe226c05..7241fc3558 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts @@ -72,6 +72,19 @@ export function downloadGroups( }); } +/** Model with this qualified id or raw name, from the listing or else the tracked downloads. */ +export function findModelOption( + key: string | null, + models: ModelOption[], + downloads: ModelQuantGroup[] +): ModelOption | null { + if (!key) return null; + + const matches = (candidate: ModelOption) => candidate.id === key || candidate.model === key; + + return models.find(matches) ?? downloads.find((group) => matches(group.base))?.base ?? null; +} + /** One collapsible block of the manager's table. */ export interface ModelsTableGroup { /** One entry per repo, its quants hanging off it. */ diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte index 33b3b2f86e..1ed74a3ea9 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte @@ -14,8 +14,9 @@ import { KeyboardKey, ServerModelStatus } from '$lib/enums'; import { useChatFormModel } from '$lib/hooks/use-chat-form-model.svelte'; import { useModelsSelector } from '$lib/hooks/use-models-selector.svelte'; - import { deviceStore, modelsStore, serverStore, settingsStore } from '$lib/stores'; + import { deviceStore, modelsStore, serverStore, settingsStore, uiStore } from '$lib/stores'; import { type ModelItem, modelLoadFraction } from '$lib/utils'; + import { untrack } from 'svelte'; interface Props { /** Model to show, when the caller owns it, e.g. the model of one message. */ @@ -165,6 +166,15 @@ } } + // the details drawer's chat actions finish the pick, so an open picker closes too + $effect(() => { + if (!uiStore.modelsSelectorCloseSignal) return; + + untrack(() => { + if (isOpen) ms.handleOpenChange(false); + }); + }); + // bits-ui auto-focuses the opened content, which can yank the page scroll: the // content prevents that and this focuses the search input instead $effect(() => { diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte index 7cc286f89d..995bfe6618 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte @@ -6,8 +6,8 @@ import { ActionIcon, ModelId } from '$lib/components/app'; import { ModelRowDownloadState } from '$lib/enums'; import { HuggingFaceService, ModelsService } from '$lib/services'; - import { deviceStore, modelsStore, uiStore } from '$lib/stores'; - import type { ModelDownloadEntry, ModelOption, ModelSidecarBadge } from '$lib/types'; + import { modelsStore, uiStore } from '$lib/stores'; + import type { ModelDownloadEntry, ModelSidecarBadge } from '$lib/types'; interface Props { /** One entry from the status feed: an in-flight or paused download. */ @@ -26,14 +26,6 @@ let pendingDrafts = $derived(sidecars.filter((badge) => badge.pending)); let onDiskDrafts = $derived(sidecars.filter((badge) => !badge.pending)); - /** The download stands for its model, so the row's info action can open on it. */ - let option = $derived({ - capabilities: [], - id: entry.repoWithTag, - model: entry.repoWithTag, - name: modelsStore.toDisplayName(entry.repoWithTag) - }); - let percent = $derived( entry.progress && entry.progress.totalBytes > 0 ? Math.round((entry.progress.downloadedBytes / entry.progress.totalBytes) * 100) @@ -81,11 +73,7 @@