mirrors/llama.cpp
0
0 Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-28 17:56:13 +08:00
Code Activity

Merge remote-tracking branch 'origin/master' into allozaur/fix/24338-new-chat-draft-deleted-on-system-prompt

Browse Source 
# Conflicts:
#	tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
#	tools/ui/src/lib/stores/chat/index.svelte.ts
This commit is contained in:
Aleksander Grygier committed 2026-10-27 07:54:04 +01:00
2 parents 2104dfdae8 de5122d879
commit 5c024cb52d
151 files changed
+8222 -3431
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/behavior-check.yml
+49
View File
@@ -107,3 +107,52 @@ jobs:
source .venv/bin/activate
cd build
ctest -L behavior -LE 'model|backend' --verbose --timeout 900
windows:
runs-on: windows-2025
cache-mode: write
permissions:
actions: write
contents: read
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.24
with:
key: behavior-check-windows-2025
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Python Dependencies
id: python_depends
shell: bash
run: |
python3 -m venv .venv
source .venv/Scripts/activate
pip install jinja2==3.1.6
- name: Build
id: cmake_build
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
cmake -B build -G "Ninja Multi-Config" ^
-DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
-DCMAKE_BUILD_TYPE=Release ^
-DGGML_NATIVE=OFF ^
-DGGML_BACKEND_DL=OFF
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
cmake --build build --config Release -j %NINJA_JOBS%
- name: Test
id: cmake_test
shell: bash
run: |
source .venv/Scripts/activate
cd build
ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
docs/backend/OPENCL.md
+2
View File
@@ -72,6 +72,8 @@ However, A6x GPUs in phones are likely not supported due to the outdated driver
| MXFP4 | Support |
| IQ4_NL | Support |
`MUL_MAT_ID` accepts an expert id of `-1` and writes zeros to that slot. `ADD_ID` adds no bias for `-1`, including in fused MoE operations.
## Model Preparation
Since common quantizations are supported now, it is recommanded to download GGUF models directly from Huggingface.
docs/backend/snapdragon/README.md
+3
View File
@@ -324,6 +324,9 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
- `GGML_HEXAGON_NHVX=0`
Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
- `GGML_HEXAGON_VTCM_SIZE=0`
Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
Enables allocating host buffers for debugging. By default, host buffers are disabled.
ggml/src/ggml-cpu/ops.cpp
+90 -210
View File
@@ -1914,213 +1914,87 @@ void ggml_compute_forward_repeat_back(
}
// ggml_compute_forward_concat
static void ggml_compute_forward_concat_any(
const ggml_compute_params * params,
ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
// rows of dst copied together, so that a strided source is read along neighboring rows
static constexpr int64_t GGML_CONCAT_TILE = 32;
const int ith = params->ith;
const int nth = params->nth;
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
GGML_ASSERT(ggml_is_contiguous_rows(src0));
GGML_ASSERT(ggml_is_contiguous_rows(src1));
int64_t o[4] = {0, 0, 0, 0};
if (dim == 0) {
GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);
} else {
o[dim] = src0->ne[dim];
}
// Region 1: copy rows from src0
for (int i3 = 0; i3 < ne03; i3++) {
for (int i2 = ith; i2 < ne02; i2 += nth) {
for (int i1 = 0; i1 < ne01; i1++) {
const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03;
char * y = ( char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;
memcpy(y, x, ggml_row_size(src0->type, ne00));
}
}
}
// Region 2: copy rows from src1, offset into dst by o[]
for (int i3 = 0; i3 < ne13; i3++) {
for (int i2 = ith; i2 < ne12; i2 += nth) {
for (int i1 = 0; i1 < ne11; i1++) {
const char * x = (const char *) src1->data + i1*nb11 + i2*nb12 + i3*nb13;
char * y = ( char *) dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0;
memcpy(y, x, ggml_row_size(src1->type, ne10));
}
// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element
template <size_t es>
static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) {
for (int64_t i0 = 0; i0 < n; i0++) {
for (int64_t r = 0; r < n_rows; r++) {
memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es);
}
}
}
static void ggml_compute_forward_concat_i8(
const ggml_compute_params * params,
ggml_tensor * dst) {
// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst
static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) {
const size_t es = ggml_type_size(src->type);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));
const int ith = params->ith;
const int nth = params->nth;
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];
const int8_t * x;
// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
} else {
x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
}
int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
*y = *x;
}
}
}
}
}
static void ggml_compute_forward_concat_f16(
const ggml_compute_params * params,
ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));
const int ith = params->ith;
const int nth = params->nth;
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];
const ggml_fp16_t * x;
// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
} else {
x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
}
ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
*y = *x;
}
}
}
}
}
static void ggml_compute_forward_concat_f32(
const ggml_compute_params * params,
ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));
const int ith = params->ith;
const int nth = params->nth;
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];
const float * x;
// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
} else {
x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
}
float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
*y = *x;
}
}
if (src->nb[0] == es) {
const size_t row_size = ggml_row_size(src->type, n);
for (int64_t r = 0; r < n_rows; r++) {
memcpy(y + r*nby, x + r*nbx, row_size);
}
return;
}
switch (es) {
case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
default: GGML_ABORT("unsupported element size");
}
}
// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1
void ggml_compute_forward_concat(
const ggml_compute_params * params,
ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
switch (src0->type) {
case GGML_TYPE_F16:
case GGML_TYPE_BF16:
case GGML_TYPE_I16:
{
ggml_compute_forward_concat_f16(params, dst);
} break;
case GGML_TYPE_I8:
{
ggml_compute_forward_concat_i8(params, dst);
} break;
case GGML_TYPE_F32:
case GGML_TYPE_I32:
{
ggml_compute_forward_concat_f32(params, dst);
} break;
default:
{
ggml_compute_forward_concat_any(params, dst);
}
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
GGML_ASSERT(nb0 == ggml_type_size(dst->type));
GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0));
GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0));
const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0;
const int64_t nr = ne1*ne2*ne3;
const int64_t dr = (nr + params->nth - 1)/params->nth;
const int64_t ir0 = dr*params->ith;
const int64_t ir1 = MIN(ir0 + dr, nr);
for (int64_t ir = ir0; ir < ir1; ) {
int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) };
// a tile of consecutive rows sharing i2, i3 and their source
int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE);
if (dim == 1 && i[1] < ne01) {
nt = MIN(nt, ne01 - i[1]);
}
char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3;
if (dim == 0 || i[dim] < src0->ne[dim]) {
ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt);
}
if (dim == 0 || i[dim] >= src0->ne[dim]) {
i[dim] -= src0->ne[dim]; // i[0] does not address rows
ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt);
}
ir += nt;
}
}
@@ -4972,8 +4846,8 @@ static void ggml_compute_forward_scale_f32(
const ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_is_contiguous(src0));
GGML_ASSERT(ggml_is_contiguous(dst));
GGML_ASSERT(ggml_is_contiguous_rows(src0));
GGML_ASSERT(ggml_is_contiguous_rows(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
float s; // scale factor
@@ -4988,6 +4862,8 @@ static void ggml_compute_forward_scale_f32(
const int nc = src0->ne[0];
const int nr = ggml_nrows(src0);
GGML_TENSOR_UNARY_OP_LOCALS
// rows per thread
const int dr = (nr + nth - 1)/nth;
@@ -4995,25 +4871,23 @@ static void ggml_compute_forward_scale_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
const size_t nb01 = src0->nb[1];
for (int i = ir0; i < ir1; i++) {
const int64_t i1 = i % ne01;
const int64_t i2 = (i / ne01) % ne02;
const int64_t i3 = i / (ne01*ne02);
const size_t nb1 = dst->nb[1];
float * y = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
if (b == 0.0f) {
for (int i1 = ir0; i1 < ir1; i1++) {
if (b == 0.0f) {
if (dst->data != src0->data) {
// src0 is same shape as dst => same indices
// TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy
memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
memcpy(y, x, nc * sizeof(float));
}
ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);
}
} else {
for (int i1 = ir0; i1 < ir1; i1++) {
ggml_vec_mad1_f32(nc,
(float *) ((char *) dst->data + i1*nb1),
(float *) ((char *) src0->data + i1*nb1),
s, b);
ggml_vec_scale_f32(nc, y, s);
} else {
ggml_vec_mad1_f32(nc, y, x, s, b);
}
}
}
@@ -5024,8 +4898,8 @@ static void ggml_compute_forward_scale_bf16(
const ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_is_contiguous(src0));
GGML_ASSERT(ggml_is_contiguous(dst));
GGML_ASSERT(ggml_is_contiguous_rows(src0));
GGML_ASSERT(ggml_is_contiguous_rows(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
float s; // scale factor
@@ -5040,6 +4914,8 @@ static void ggml_compute_forward_scale_bf16(
const int nc = src0->ne[0];
const int nr = ggml_nrows(src0);
GGML_TENSOR_UNARY_OP_LOCALS
// rows per thread
const int dr = (nr + nth - 1)/nth;
@@ -5047,9 +4923,13 @@ static void ggml_compute_forward_scale_bf16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
for (int i1 = ir0; i1 < ir1; i1++) {
const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]);
ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*dst->nb[1]);
for (int i = ir0; i < ir1; i++) {
const int64_t i1 = i % ne01;
const int64_t i2 = (i / ne01) % ne02;
const int64_t i3 = i / (ne01*ne02);
const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
for (int k = 0; k < nc; k++) {
y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
}
ggml/src/ggml-cuda/ggml-cuda.cu
+7 -1
View File
@@ -3822,6 +3822,11 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
return false;
}
// TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense
if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) {
return false;
}
if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) {
return false;
}
@@ -5972,7 +5977,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_LOG:
return true;
case GGML_OP_SCALE:
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type;
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_CLAMP:
return ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_ADD:
ggml/src/ggml-cuda/scale.cu
+33 -10
View File
@@ -3,23 +3,45 @@
#define MAX_GRIDDIM_X 0x7FFFFFFF
// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements).
template <typename T>
static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) {
static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k,
const uint3 ne0, const uint3 ne1, const uint3 ne2,
const uint32_t s01, const uint32_t s02, const uint32_t s03,
const uint32_t s1, const uint32_t s2, const uint32_t s3) {
ggml_cuda_pdl_lc();
int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;
int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;
const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
ggml_cuda_pdl_sync();
for (int64_t i = tid; i < nelements; i += stride) {
dst[i] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i]) + bias);
if (i >= k) {
return;
}
const uint2 d0 = fast_div_modulo(i, ne0); // <i / ne0, i0>
const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1>
const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2>
const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03;
const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3;
dst[i_dst] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i_src]) + bias);
}
template <typename T>
static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {
const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) {
const int64_t k = ggml_nelements(src0);
const size_t ts = sizeof(T);
GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max());
const uint3 ne0 = init_fastdiv_values(src0->ne[0]);
const uint3 ne1 = init_fastdiv_values(src0->ne[1]);
const uint3 ne2 = init_fastdiv_values(src0->ne[2]);
const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream);
ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, nelements);
ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2,
src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts,
t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts);
}
void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -28,6 +50,7 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16);
GGML_ASSERT(dst->type == src0->type);
GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst));
float scale;
float bias;
@@ -35,8 +58,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));
if (src0->type == GGML_TYPE_BF16) {
scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream);
scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream);
} else {
scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream);
scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream);
}
}
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+12 -2
View File
@@ -85,6 +85,7 @@ int opt_arch = 0; // autodetect
size_t opt_ndev = 1;
size_t opt_nhvx = 0; // use all
int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only
size_t opt_vtcm_size = 0; // VTCM size override in bytes; 0 = auto-detect
size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings
int opt_etm = 0;
int opt_verbose = 0;
@@ -2152,6 +2153,12 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->n_hvx = this->n_threads;
this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
this->vtcm_size = (uint64_t)hw_vtcm_size;
if (opt_vtcm_size > 0) {
if (opt_vtcm_size > hw_vtcm_size) {
GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size);
}
this->vtcm_size = opt_vtcm_size;
}
GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
(unsigned long long)(this->vtcm_size / (1024 * 1024)));
@@ -2161,7 +2168,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
this->n_hvx = this->n_threads;
this->n_hmx = (opt_nhmx != 0) ? 1 : 0;
this->vtcm_size = 8 * 1024 * 1024;
this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
}
}
@@ -2230,7 +2237,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
// Start dspqueue/opbatch processing
err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem,
(uint32_t) opt_power_mode);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
@@ -3391,6 +3399,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
const char * str_etm = getenv("GGML_HEXAGON_ETM");
const char * str_nhvx = getenv("GGML_HEXAGON_NHVX");
const char * str_nhmx = getenv("GGML_HEXAGON_NHMX");
const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE");
const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT");
@@ -3456,6 +3465,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
}
opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx;
opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx;
opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size) : opt_vtcm_size;
opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;
opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;
opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split;
ggml/src/ggml-hexagon/htp/htp_iface.idl
+1 -1
View File
@@ -11,7 +11,7 @@ struct htp_iface_pmu_conf {
};
interface htp_iface : remote_handle64 {
AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);
AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode);
AEEResult stop();
AEEResult mmap(in uint32 fd, in uint64 size);
AEEResult munmap(in uint32 fd);
ggml/src/ggml-hexagon/htp/main.c
+6 -3
View File
@@ -325,8 +325,11 @@ static unsigned int vtcm_query_size(void) {
return vtcm_size;
}
static int vtcm_alloc(struct htp_context * ctx) {
static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) {
unsigned int vtcm_size = vtcm_query_size();
if (size_override) {
vtcm_size = (unsigned int) size_override;
}
compute_res_attr_t attr;
HAP_compute_res_attr_init(&attr);
@@ -438,7 +441,7 @@ static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] =
#undef HTP_POWER_PROFILE_ENTRY
AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) {
AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h) {
return AEE_EBADPARM;
@@ -557,7 +560,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
ctx->max_vmem = max_vmem;
ctx->dsp_queue = dsp_queue;
err = vtcm_alloc(ctx);
err = vtcm_alloc(ctx, vtcm_size);
if (err != AEE_SUCCESS) {
FARF(ERROR, "Unable to allocate VTCM");
htp_iface_stop(handle);
ggml/src/ggml-hexagon/htp/matmul-ops.c
+159 -91
View File
@@ -16,6 +16,7 @@
#include "hvx-dump.h"
#include "hvx-arith.h"
#include "hvx-reduce.h"
#include "hvx-copy.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
@@ -1413,7 +1414,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert
const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]);
if (eid == -1) {
if (eid < 0) {
continue;
}
assert(eid >= 0 && eid < (int32_t) n_ids);
@@ -4111,7 +4112,7 @@ static int hvx_mm_matmul_id_nx(
return HTP_STATUS_OK;
}
static inline void scan_expert_ids_n(
static __attribute__((noinline)) uint32_t scan_expert_ids_n(
const struct htp_tensor * ids,
const uint32_t n_ids,
uint32_t n_as,
@@ -4121,15 +4122,17 @@ static inline void scan_expert_ids_n(
) {
const size_t ids_nb1 = ids->nb[1];
const uint8_t * ids_data = (const uint8_t *) ids->data;
uint32_t n_skipped = 0;
for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
for (uint32_t id = 0; id < n_ids; ++id) {
const int32_t i02 = row_ptr[id];
if (i02 == -1) {
if (i02 < 0) {
n_skipped++;
continue;
}
assert(i02 >= 0 && i02 < n_as);
assert(i02 >= 0 && i02 < (int32_t) n_as);
if (matrix_rows) {
matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4137,31 +4140,10 @@ static inline void scan_expert_ids_n(
counts[i02] += 1;
}
}
return n_skipped;
}
// the matmuls only write the rows that an expert owns, so the rows of the skipped slots are zeroed here
static inline void zero_skipped_dst_rows(
const struct htp_tensor * ids,
uint32_t n_ids,
const struct htp_tensor * dst,
uint32_t ne0,
size_t nb1,
size_t nb2
) {
const uint8_t * ids_data = (const uint8_t *) ids->data;
for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
for (uint32_t id = 0; id < n_ids; ++id) {
const int32_t i02 = *(const int32_t *) (ids_data + iid1 * ids->nb[1] + id * ids->nb[0]);
if (i02 != -1) {
continue;
}
memset((uint8_t *) (dst->data + id * nb1 + iid1 * nb2), 0, ne0 * sizeof(float));
}
}
}
static inline void scan_expert_ids(
static __attribute__((noinline)) uint32_t scan_expert_ids(
const struct htp_tensor * ids,
uint32_t n_ids,
uint32_t n_as,
@@ -4173,23 +4155,25 @@ static inline void scan_expert_ids(
if (ids_nb0 == 4) {
switch (n_ids) {
case 8: scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); break;
case 4: scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); break;
case 2: scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); break;
default: scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); break;
case 8: return scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride);
case 4: return scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride);
case 2: return scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride);
default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride);
}
} else {
// Strided fallback
const size_t ids_nb1 = ids->nb[1];
const uint8_t * ids_data = (const uint8_t *) ids->data;
uint32_t n_skipped = 0;
for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
for (uint32_t id = 0; id < n_ids; ++id) {
const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0);
if (i02 == -1) {
if (i02 < 0) {
n_skipped++;
continue;
}
assert(i02 >= 0 && i02 < n_as);
assert(i02 >= 0 && i02 < (int32_t) n_as);
if (matrix_rows) {
matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4197,9 +4181,124 @@ static inline void scan_expert_ids(
counts[i02] += 1;
}
}
return n_skipped;
}
}
static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) {
const uint8_t * ids_data = (const uint8_t *) ids->data;
const size_t ids_nb0 = ids->nb[0];
uint32_t n_skipped = 0;
for (uint32_t id = 0; id < n_ids; ++id) {
const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0);
if (i02 < 0) {
n_skipped++;
}
}
return n_skipped;
}
// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns.
static void zero_skipped_dst_rows(
struct htp_ops_context * octx,
const struct htp_tensor * ids,
uint32_t n_ids,
const struct htp_tensor * dst
) {
const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float);
uint32_t col_start = 0;
uint32_t col_count = dst->ne[0];
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, elem_size);
const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
col_start = range.start;
col_count = range.count;
}
if (col_count == 0) {
return;
}
const uint8_t * ids_data = (const uint8_t *) ids->data;
const size_t ids_nb0 = ids->nb[0];
const size_t ids_nb1 = ids->nb[1];
const size_t dst_nb1 = dst->nb[1];
const size_t dst_nb2 = dst->nb[2];
const uint32_t col_offset_bytes = col_start * elem_size;
for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
const uint8_t * ids_row = ids_data + iid1 * ids_nb1;
uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes;
for (uint32_t id = 0; id < n_ids; ++id) {
const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0);
if (i02 < 0) {
hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size);
}
}
}
}
static int htp_mmid_prepare_mapping(
struct htp_ops_context * octx,
const struct htp_tensor * ids,
uint32_t n_ids,
uint32_t n_as,
uint32_t act_nrows,
uint8_t ** out_mapping_buf,
uint32_t ** out_matrix_row_counts,
struct mmid_row_mapping ** out_matrix_rows,
uint32_t * out_mapping_stride,
uint32_t * out_n_skipped
) {
uint8_t * mapping_buf = octx->ctx->ddr_spad_base;
uint32_t mapping_stride = 1;
uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
struct mmid_row_mapping * matrix_rows = NULL;
uint32_t n_skipped = 0;
if (act_nrows > 1) {
const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
mapping_stride = max_count > 0 ? max_count : 1;
size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
if (total_map_size > octx->ctx->ddr_spad_size) {
mapping_buf = memalign(128, total_map_size);
if (!mapping_buf) {
return HTP_STATUS_INTERNAL_ERR;
}
}
matrix_row_counts = (uint32_t *) mapping_buf;
matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
} else {
n_skipped = count_skipped_ids_single(ids, n_ids);
}
*out_mapping_buf = mapping_buf;
*out_matrix_row_counts = matrix_row_counts;
*out_matrix_rows = matrix_rows;
*out_mapping_stride = mapping_stride;
*out_n_skipped = n_skipped;
return HTP_STATUS_OK;
}
int op_matmul_id(struct htp_ops_context * octx) {
htp_matmul_tensors_preamble;
@@ -4235,41 +4334,22 @@ int op_matmul_id(struct htp_ops_context * octx) {
const int n_ids = ids->ne[0]; // n_expert_used
const int n_as = ne02; // n_expert
uint8_t * mapping_buf = octx->ctx->ddr_spad_base;
uint32_t mapping_stride = 1;
uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
uint8_t * mapping_buf = NULL;
uint32_t * matrix_row_counts = NULL;
struct mmid_row_mapping * matrix_rows = NULL;
uint32_t mapping_stride = 1;
uint32_t n_skipped = 0;
if (act_nrows > 1) {
const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
memset(matrix_row_counts, 0, matrix_row_counts_size);
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
mapping_stride = max_count > 0 ? max_count : 1;
size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
if (total_map_size > octx->ctx->ddr_spad_size) {
mapping_buf = memalign(128, total_map_size);
if (!mapping_buf) {
return HTP_STATUS_INTERNAL_ERR;
}
}
matrix_row_counts = (uint32_t *) mapping_buf;
matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
&mapping_buf, &matrix_row_counts, &matrix_rows,
&mapping_stride, &n_skipped);
if (map_status != HTP_STATUS_OK) {
return map_status;
}
zero_skipped_dst_rows(ids, n_ids, dst, ne0, nb1, nb2);
if (n_skipped > 0) {
zero_skipped_dst_rows(octx, ids, n_ids, dst);
}
mmctx->matrix_row_counts = matrix_row_counts;
mmctx->matrix_rows = matrix_rows;
@@ -4360,38 +4440,26 @@ int op_matmul_id_nx(struct htp_ops_context * octx) {
const int n_ids = ids->ne[0];
const int n_as = src0->ne[2];
uint8_t * mapping_buf = octx->ctx->ddr_spad_base;
uint32_t mapping_stride = 1;
uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
uint8_t * mapping_buf = NULL;
uint32_t * matrix_row_counts = NULL;
struct mmid_row_mapping * matrix_rows = NULL;
uint32_t mapping_stride = 1;
uint32_t n_skipped = 0;
if (act_nrows > 1) {
const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
&mapping_buf, &matrix_row_counts, &matrix_rows,
&mapping_stride, &n_skipped);
if (map_status != HTP_STATUS_OK) {
return map_status;
}
hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
memset(matrix_row_counts, 0, matrix_row_counts_size);
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
mapping_stride = max_count > 0 ? max_count : 1;
size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
if (total_map_size > octx->ctx->ddr_spad_size) {
mapping_buf = memalign(128, total_map_size);
if (!mapping_buf) {
return HTP_STATUS_INTERNAL_ERR;
if (n_skipped > 0) {
for (uint32_t p = 0; p < n_weights; ++p) {
const struct htp_tensor * restrict d = octx->dsts[p];
if (d) {
zero_skipped_dst_rows(octx, ids, n_ids, d);
}
}
matrix_row_counts = (uint32_t *) mapping_buf;
matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
}
mmctx->matrix_row_counts = matrix_row_counts;
ggml/src/ggml-metal/ggml-metal-device.m
+1
View File
@@ -1555,6 +1555,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
switch (op->op) {
case GGML_OP_SCALE:
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16);
case GGML_OP_FILL:
case GGML_OP_CLAMP:
case GGML_OP_SQR:
ggml/src/ggml-metal/kernels/unary.metal
+51 -29
View File
@@ -3,6 +3,24 @@
constant short FC_unary_op [[function_constant(FC_UNARY + 0)]];
constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]];
template <typename T, typename U>
inline constexpr T ggml_metal_unary_cast(U x) {
return (T) x;
}
#if defined(GGML_METAL_HAS_BF16)
// MSL has no C-style cast from float vectors to bfloat vectors
template <>
inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3]));
}
template <>
inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x));
}
#endif
template <typename T0, typename T, typename TC>
kernel void kernel_unary_impl(
constant ggml_metal_kargs_unary & args,
@@ -48,79 +66,79 @@ kernel void kernel_unary_impl(
const TC x = (TC) src0_ptr[i0];
if (FC_OP == OP_UNARY_NUM_SCALE) {
dst_ptr[i0] = (T) (args.scale * x + args.bias);
dst_ptr[i0] = ggml_metal_unary_cast<T>(args.scale * x + args.bias);
}
if (FC_OP == OP_UNARY_NUM_FILL) {
dst_ptr[i0] = (T) args.val;
dst_ptr[i0] = ggml_metal_unary_cast<T>(args.val);
}
if (FC_OP == OP_UNARY_NUM_CLAMP) {
dst_ptr[i0] = (T) clamp(x, args.min, args.max);
dst_ptr[i0] = ggml_metal_unary_cast<T>(clamp(x, args.min, args.max));
}
if (FC_OP == OP_UNARY_NUM_SQR) {
dst_ptr[i0] = (T) (x * x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(x * x);
}
if (FC_OP == OP_UNARY_NUM_SQRT) {
dst_ptr[i0] = (T) sqrt(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(sqrt(x));
}
if (FC_OP == OP_UNARY_NUM_SIN) {
dst_ptr[i0] = (T) sin(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(sin(x));
}
if (FC_OP == OP_UNARY_NUM_COS) {
dst_ptr[i0] = (T) cos(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(cos(x));
}
if (FC_OP == OP_UNARY_NUM_LOG) {
dst_ptr[i0] = (T) log(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(log(x));
}
if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) {
dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
dst_ptr[i0] = ggml_metal_unary_cast<T>(TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
}
if (FC_OP == OP_UNARY_NUM_TANH) {
dst_ptr[i0] = (T) precise::tanh(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(precise::tanh(x));
}
if (FC_OP == OP_UNARY_NUM_RELU) {
dst_ptr[i0] = (T) fmax(0, x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, x));
}
if (FC_OP == OP_UNARY_NUM_SIGMOID) {
dst_ptr[i0] = (T) (1 / (1 + exp(-x)));
dst_ptr[i0] = ggml_metal_unary_cast<T>(1 / (1 + exp(-x)));
}
if (FC_OP == OP_UNARY_NUM_GELU) {
dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
}
if (FC_OP == OP_UNARY_NUM_GELU_ERF) {
dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
}
if (FC_OP == OP_UNARY_NUM_GELU_QUICK) {
dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x))));
dst_ptr[i0] = ggml_metal_unary_cast<T>(x * (1/(1 + exp(GELU_QUICK_COEF*x))));
}
if (FC_OP == OP_UNARY_NUM_SILU) {
dst_ptr[i0] = (T) (x / (1 + exp(-x)));
dst_ptr[i0] = ggml_metal_unary_cast<T>(x / (1 + exp(-x)));
}
if (FC_OP == OP_UNARY_NUM_ELU) {
dst_ptr[i0] = (T) elu_approx(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(elu_approx(x));
}
if (FC_OP == OP_UNARY_NUM_NEG) {
dst_ptr[i0] = (T) -x;
dst_ptr[i0] = ggml_metal_unary_cast<T>(-x);
}
if (FC_OP == OP_UNARY_NUM_ABS) {
dst_ptr[i0] = (T) fabs(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(fabs(x));
}
if (FC_OP == OP_UNARY_NUM_SGN) {
@@ -132,40 +150,40 @@ kernel void kernel_unary_impl(
}
if (FC_OP == OP_UNARY_NUM_HARDSWISH) {
dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5)));
dst_ptr[i0] = ggml_metal_unary_cast<T>(x * fmax(0, fmin(1, x/6 + 0.5)));
}
if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) {
dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5));
dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, fmin(1, x/6 + 0.5)));
}
if (FC_OP == OP_UNARY_NUM_EXP) {
dst_ptr[i0] = (T) exp(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x));
}
if (FC_OP == OP_UNARY_NUM_SOFTPLUS) {
dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20);
dst_ptr[i0] = ggml_metal_unary_cast<T>(select(log(1 + exp(x)), x, x > 20));
}
if (FC_OP == OP_UNARY_NUM_EXPM1) {
// TODO: precise implementation
dst_ptr[i0] = (T) (exp(x) - 1);
dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x) - 1);
}
if (FC_OP == OP_UNARY_NUM_FLOOR) {
dst_ptr[i0] = (T) floor(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(floor(x));
}
if (FC_OP == OP_UNARY_NUM_CEIL) {
dst_ptr[i0] = (T) ceil(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(ceil(x));
}
if (FC_OP == OP_UNARY_NUM_ROUND) {
dst_ptr[i0] = (T) round(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(round(x));
}
if (FC_OP == OP_UNARY_NUM_TRUNC) {
dst_ptr[i0] = (T) trunc(x);
dst_ptr[i0] = ggml_metal_unary_cast<T>(trunc(x));
}
if (FC_OP == OP_UNARY_NUM_XIELU) {
@@ -174,7 +192,7 @@ kernel void kernel_unary_impl(
const TC clamped = fmin(xi, TC(args.val));
const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi;
const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi;
dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg);
dst_ptr[i0] = ggml_metal_unary_cast<T>(gate * y_pos + (TC(1.0f) - gate) * y_neg);
}
}
@@ -188,6 +206,10 @@ template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_un
template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl<half, half, float>;
template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4, half4, float4>;
#if defined(GGML_METAL_HAS_BF16)
template [[host_name("kernel_unary_bf16_bf16")]] kernel kernel_unary_t kernel_unary_impl<bfloat, bfloat, float>;
template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, bfloat4, float4>;
#endif
kernel void kernel_silu_back_f32(
constant ggml_metal_kargs_silu_back & args,
ggml/src/ggml-opencl/CMakeLists.txt
+7
View File
@@ -104,7 +104,13 @@ set(GGML_OPENCL_KERNELS
gemm_noshuffle_q2_k_q8_1_dp4a
gemv_noshuffle_q2_k_f32
gemm_noshuffle_q2_k_f32
gemm_noshuffle_q3_k_f32
gemm_noshuffle_q3_k_q8_1_dp4a
gemv_noshuffle_q3_k_f32
mul_mv_q2_k_f32
mul_mv_q3_k_f32
mul_mv_q3_k_f32_flat
mul_mm_q3_k_f32_l4_lm
mul_mv_q2_k_f32_flat
mul_mm_q2_k_f32_l4_lm
mul_mv_q4_k_f32
@@ -128,6 +134,7 @@ set(GGML_OPENCL_KERNELS
mul_mv_id_q8_0_f32_flat
mul_mv_id_mxfp4_f32
mul_mv_id_mxfp4_f32_flat
mul_mat_id_utils
gemm_moe_q4_0_f32_ns
gemm_moe_q4_0_q8_1_dp4a
gemv_moe_q4_0_f32_ns
ggml/src/ggml-opencl/ggml-opencl.cpp
+686 -32
View File
@@ -975,6 +975,17 @@ struct ggml_backend_opencl_context {
int q2_k_mv_rows = 4;
int q2_k_mv_nsg = 4;
cl_kernel kernel_mul_mv_q2_K_f32;
cl_kernel kernel_convert_block_q3_K, kernel_restore_block_q3_K;
cl_kernel kernel_mul_mv_q3_K_f32_flat;
cl_kernel kernel_mul_mm_q3_k_f32_l4_lm;
cl_kernel kernel_convert_block_q3_k_noshuffle, kernel_restore_block_q3_k_noshuffle;
cl_kernel kernel_gemv_noshuffle_q3_k_f32 = nullptr;
cl_kernel kernel_gemm_noshuffle_q3_k_f32 = nullptr;
cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a = nullptr;
cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = nullptr;
int q3_k_mv_rows = 2;
int q3_k_mv_nsg = 8;
cl_kernel kernel_mul_mv_q3_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32_flat;
cl_kernel kernel_mul_mv_q5_K_f32;
@@ -1542,6 +1553,23 @@ static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * b
return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0);
}
static std::string ggml_cl_q3_k_mm_options(const ggml_backend_opencl_context * backend_ctx) {
const bool x2 = backend_ctx->gpu_family == ADRENO &&
(backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
int bk = x2 ? 16 : 32;
bool half = x2;
if (const char * env = getenv("GGML_OPENCL_LM_BK")) {
const int value = atoi(env);
if (value == 16 || value == 32) {
bk = value;
}
}
if (const char * env = getenv("GGML_OPENCL_LM_HALF")) {
half = atoi(env) != 0;
}
return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0);
}
static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) {
// compiler options for general kernels
auto opencl_c_std =
@@ -1815,6 +1843,21 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// mul_mat_id_utils
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mat_id_utils.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mat_id_utils.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// moe_add_id_glu
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -1992,6 +2035,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err));
@@ -2398,6 +2445,53 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// mul_mv_q3_k_f32_flat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mv_q3_k_f32_flat.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mv_q3_k_f32_flat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32_flat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mm_q3_k_f32_l4_lm
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mm_q3_k_f32_l4_lm.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mm_q3_k_f32_l4_lm.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q3_k_mm_options(backend_ctx));
CL_CHECK((backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q3_k_f32_l4_lm", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mv_q3_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mv_q3_k_f32.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mv_q3_k_f32.cl");
#endif
cl_program prog =
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mv_q4_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4206,6 +4300,80 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// gemm_noshuffle_q3_k_q8_1_dp4a
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q3_k_q8_1_dp4a.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q3_k_q8_1_dp4a.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8");
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// gemm_noshuffle_q3_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q3_k_f32.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// gemv_noshuffle_q3_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemv_noshuffle_q3_k_f32.cl.h"
};
#else
const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl");
#endif
if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) {
const int rows = atoi(env);
if (rows == 1 || rows == 2 || rows == 4) {
backend_ctx->q3_k_mv_rows = rows;
}
}
if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) {
const int nsg = atoi(env);
if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
backend_ctx->q3_k_mv_nsg = nsg;
}
}
for (;;) {
const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) +
" -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg);
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err);
CL_CHECK(err);
CL_CHECK(clReleaseProgram(prog));
size_t max_workgroup;
CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) {
backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel;
break;
}
CL_CHECK(clReleaseKernel(kernel));
GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1);
backend_ctx->q3_k_mv_nsg /= 2;
}
GGML_LOG_CONT(".");
}
// gemm_noshuffle_q2_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -5737,7 +5905,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err));
CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err));
CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));
CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));
CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));
@@ -8245,6 +8412,27 @@ struct ggml_tensor_extra_cl_q2_K {
}
};
struct ggml_tensor_extra_cl_q3_K {
bool noshuffle = false;
cl_mem qs = nullptr; // quantized values
cl_mem hm = nullptr; // high bits
cl_mem sc = nullptr; // packed 6-bit scales
cl_mem d = nullptr; // super-block scales
size_t size_qs = 0, size_hm = 0, size_sc = 0, size_d = 0;
~ggml_tensor_extra_cl_q3_K() { reset(); }
void reset() {
noshuffle = false;
if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
if (hm != nullptr) { CL_CHECK(clReleaseMemObject(hm)); hm = nullptr; }
if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; }
if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
size_qs = size_hm = size_sc = size_d = 0;
}
};
struct ggml_tensor_extra_cl_q6_K {
// Lower 4 bits of quantized weights.
cl_mem ql = nullptr;
@@ -9345,6 +9533,24 @@ static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backe
#endif
}
static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
if (const char * env = getenv("GGML_OPENCL_Q3_K_NOSHUFFLE")) {
if (env[0] == '0') {
return false;
}
}
return tensor->type == GGML_TYPE_Q3_K &&
backend_ctx->gpu_family == ADRENO &&
ggml_is_contiguous(tensor) && !tensor->view_src &&
use_adreno_kernels(backend_ctx, tensor);
#else
GGML_UNUSED(backend_ctx);
GGML_UNUSED(tensor);
return false;
#endif
}
inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
// The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*.
// Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights.
@@ -9868,6 +10074,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
} else if (op->src[0]->type == GGML_TYPE_Q4_0) {
// Non-contig src0 routes through on-device dequant-to-f16.
return op->src[1]->type == GGML_TYPE_F32;
} else if (op->src[0]->type == GGML_TYPE_Q3_K) {
const ggml_tensor * weight = op->src[0]->view_src;
if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) {
return false;
}
return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_Q2_K) {
const ggml_tensor * weight = op->src[0]->view_src;
if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) {
@@ -10254,12 +10466,18 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) {
delete e;
}
for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) {
delete e;
}
for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {
delete e;
}
for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) {
delete e;
}
for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) {
delete e;
}
for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) {
delete e;
}
@@ -10448,6 +10666,21 @@ struct ggml_backend_opencl_buffer_context {
return extra;
}
ggml_tensor_extra_cl_q3_K * ggml_opencl_alloc_temp_tensor_extra_q3_K() {
ggml_tensor_extra_cl_q3_K * extra;
if (temp_tensor_extras_q3_K.empty()) {
extra = new ggml_tensor_extra_cl_q3_K();
} else {
extra = temp_tensor_extras_q3_K.back();
temp_tensor_extras_q3_K.pop_back();
}
temp_tensor_extras_q3_K_in_use.push_back(extra);
extra->reset();
return extra;
}
ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() {
ggml_tensor_extra_cl_q6_K * extra;
if (temp_tensor_extras_q6_K.empty()) {
@@ -10529,6 +10762,11 @@ struct ggml_backend_opencl_buffer_context {
}
temp_tensor_extras_q2_K_in_use.clear();
for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) {
temp_tensor_extras_q3_K.push_back(e);
}
temp_tensor_extras_q3_K_in_use.clear();
q8_0_soa_tensors.clear();
q4_0_soa_tensors.clear();
}
@@ -10561,7 +10799,9 @@ struct ggml_backend_opencl_buffer_context {
std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K;
std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use;
std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K;
std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K;
std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K_in_use;
std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K_in_use;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use;
@@ -10818,6 +11058,91 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
return;
}
if (tensor->type == GGML_TYPE_Q3_K) {
GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
ggml_tensor_extra_cl_q3_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q3_K();
extra->noshuffle = ggml_cl_q3_k_use_noshuffle(backend_ctx, tensor);
const size_t blck = (size_t)ggml_blck_size(tensor->type);
const size_t n_blk = ggml_nelements(tensor) / blck;
const size_t size_qs = n_blk * (blck/4);
const size_t size_hm = n_blk * (blck/8);
const size_t size_sc = n_blk * 12;
const size_t size_d = n_blk * sizeof(ggml_fp16_t);
GGML_ASSERT(size_qs + size_hm + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
ggml_nbytes(tensor), NULL, &err);
CL_CHECK(err);
CL_CHECK(clEnqueueWriteBuffer(
queue, data_device, CL_TRUE, 0,
ggml_nbytes(tensor), data, 0, NULL, NULL));
cl_buffer_region region;
region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
region.size = size_qs;
CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
auto previous_origin = region.origin;
region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment);
region.size = size_hm;
CL_CHECK((extra->hm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
previous_origin = region.origin;
region.origin = align_to(previous_origin + size_hm, backend_ctx->alignment);
region.size = size_sc;
CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
previous_origin = region.origin;
region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment);
region.size = size_d;
CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q3_k_noshuffle
: backend_ctx->kernel_convert_block_q3_K;
cl_ulong nb_arg = (cl_ulong)n_blk;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->hm));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sc));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg));
size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
size_t local_work_size[] = { 64, 1, 1 };
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
CL_CHECK(clWaitForEvents(1, &evt));
CL_CHECK(clReleaseEvent(evt));
CL_CHECK(clReleaseMemObject(data_device));
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (extra->noshuffle) {
const int m = tensor->ne[1];
const int k = tensor->ne[0];
transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true);
transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true);
transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true);
transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true);
}
#endif
extra->size_qs = size_qs;
extra->size_hm = size_hm;
extra->size_sc = size_sc;
extra->size_d = size_d;
tensor->extra = extra;
return;
}
if (tensor->type == GGML_TYPE_Q1_0) {
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
@@ -10901,7 +11226,16 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
// conversion kernel.
if (tensor->type == GGML_TYPE_Q4_0) {
// Views can't SoA-ify here — parent owns the layout (see q8_0 guard).
// A view of an AoS tensor (the KV cache, written by a state restore)
// is written in place.
if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
if (!ggml_cl_is_q4_0_soa(tensor)) {
ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
CL_CHECK(clEnqueueWriteBuffer(
queue, extra_aos->data_device, CL_TRUE,
extra_aos->offset + tensor->view_offs + offset,
size, data, 0, NULL, NULL));
}
return;
}
// Tensors should have been preallocated, therefore they should
@@ -11605,7 +11939,16 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
}
if (tensor->type == GGML_TYPE_Q8_0) {
// Views share the parent's buffer; parent owns SoA conversion.
// A view of an AoS tensor (the KV cache, written by a state restore)
// is written in place.
if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
if (!ggml_cl_is_q8_0_soa(tensor)) {
ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
CL_CHECK(clEnqueueWriteBuffer(
queue, extra_aos->data_device, CL_TRUE,
extra_aos->offset + tensor->view_offs + offset,
size, data, 0, NULL, NULL));
}
return;
}
@@ -12440,7 +12783,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
GGML_ASSERT(extra);
cl_ulong n_elements = size / sizeof(ggml_fp16_t);
cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t);
cl_ulong off_dst = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t);
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR,
@@ -12468,8 +12811,9 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
GGML_ASSERT(extra);
// a view reuses its parent's extra, whose offset does not include view_offs
CL_CHECK(clEnqueueWriteBuffer(
queue, extra->data_device, CL_TRUE, extra->offset + offset,
queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset,
size, data, 0, NULL, NULL));
GGML_UNUSED(buffer);
@@ -12534,6 +12878,56 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
return;
}
if (tensor->type == GGML_TYPE_Q3_K) {
ggml_tensor_extra_cl_q3_K * extra = (ggml_tensor_extra_cl_q3_K *)tensor->extra;
const size_t blck = (size_t)ggml_blck_size(tensor->type);
ggml_cl_buffer buf_unpacked;
buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
cl_ulong nb_arg = (cl_ulong)n_blk;
cl_mem qs = extra->qs, hm = extra->hm, sc = extra->sc, d = extra->d;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_buffer buf_qs, buf_hm, buf_sc, buf_d;
if (extra->noshuffle) {
buf_qs.allocate(context, extra->size_qs);
buf_hm.allocate(context, extra->size_hm);
buf_sc.allocate(context, extra->size_sc);
buf_d.allocate(context, extra->size_d);
qs = buf_qs.buffer;
hm = buf_hm.buffer;
sc = buf_sc.buffer;
d = buf_d.buffer;
const int m = tensor->ne[1];
const int k = tensor->ne[0];
transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true);
transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true);
transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true);
transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true);
}
#endif
cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle
: backend_ctx->kernel_restore_block_q3_K;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hm));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg));
size_t gws[] = { (n_blk + 63) / 64 * 64 };
size_t lws[] = { 64 };
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
CL_CHECK(clWaitForEvents(1, &evt));
CL_CHECK(clReleaseEvent(evt));
CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
return;
}
if (tensor->type == GGML_TYPE_Q1_0) {
ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;
@@ -24905,6 +25299,65 @@ static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor
return true;
}
static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int m = src0->ne[1];
const int n = src1->ne[1];
const int k = src0->ne[0];
const int tile_n = n <= 16 ? 8 : 32;
cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow
: backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a;
const char * env = getenv("GGML_OPENCL_Q3_K_DENSE_DP4A");
if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
return false;
}
cl_int err;
cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const size_t n_blocks = (size_t)n*(k/32);
backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
cl_int nb = n_blocks;
cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
size_t q_local[] = { 64 };
backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->hm));
CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sc));
CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n));
CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k));
size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
size_t local[] = { 64, 1, 1 };
backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
CL_CHECK(clReleaseMemObject(activation));
return true;
}
static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
@@ -25004,6 +25457,108 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int m = src0->ne[1];
const int n = src1->ne[1];
const int k = src0->ne[0];
if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32;
const int stride_d = dst->ne[0];
const int stride_b = src1->nb[1]/sizeof(float);
CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm));
CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc));
CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k));
CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b));
CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d));
const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows;
const size_t nsg = backend_ctx->q3_k_mv_nsg;
size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
size_t local[] = { 64, nsg, 1 };
backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
return;
}
if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) {
return;
}
cl_context context = backend_ctx->context;
cl_int err;
cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
cl_image_desc img_desc = {};
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
img_desc.image_width = (size_t)k*n/4;
img_desc.buffer = b_sub_buf;
cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
CL_CHECK(err);
const int padded_n = CEIL_DIV(n, 8)*8;
region.origin = 0;
region.size = (size_t)k*padded_n*sizeof(cl_half);
backend_ctx->prealloc_act_trans.allocate(context, region.size);
cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
img_fmt = { CL_RGBA, CL_HALF_FLOAT };
img_desc.image_width = (size_t)k*padded_n/4;
img_desc.buffer = b_sub_buf_trans;
cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
CL_CHECK(err);
const int height_b = std::max(n/4, 1);
const int width_b = k/4;
const int padded_height_b = padded_n/4;
cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
size_t t_local[] = { 1, 16 };
backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n));
size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
size_t local[] = { 1, 128, 1 };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
CL_CHECK(clReleaseMemObject(b_img_trans));
CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
CL_CHECK(clReleaseMemObject(b_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
#endif
static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@@ -25075,6 +25630,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra;
ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra;
ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra;
ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra;
ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra;
ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra;
ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra;
@@ -25476,6 +26032,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
return;
}
// q3_k x fp32
if (src0t == GGML_TYPE_Q3_K && src1t == GGML_TYPE_F32 && extra0_q3_K->noshuffle) {
ggml_cl_mul_mat_q3_k_f32_adreno(backend, src0, src1, dst);
return;
}
// q4_k x fp32
if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst);
@@ -26026,6 +26588,51 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
case GGML_TYPE_Q3_K: {
if (ne11 < 32) {
break;
}
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
break;
}
kernel = backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm;
nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
offset0 = src0->view_offs;
int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
int batch_stride_d = ne0*ne1;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b));
CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d));
CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3));
size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
size_t local_work_size[] = {(size_t)nth0, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
case GGML_TYPE_Q4_K: {
if (ne11 < 32) {
break;
@@ -27005,7 +27612,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
#endif // GGML_OPENCL_SOA_Q
break;
}
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q3_K: {
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
ndst = 2;
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 1;
ndst = 2;
} else {
GGML_ASSERT(false && "TODO: Unknown GPU");
}
#ifdef GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_q3_K_f32_flat;
offset0 = src0->view_offs;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3));
#else
kernel = backend_ctx->kernel_mul_mv_q3_K_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
#endif // GGML_OPENCL_SOA_Q
break;
}
case GGML_TYPE_Q4_K: {
#ifdef GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat;
@@ -27313,7 +27983,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
src0t == GGML_TYPE_Q8_0 ||
src0t == GGML_TYPE_Q1_0 ||
src0t == GGML_TYPE_IQ4_NL ||
src0t == GGML_TYPE_Q2_K) {
src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K) {
// Each SIMD group produces N_DST values in the result. Assuming each
// workgroup has N_SIMDGROUP SIMD groups, then each workgroup will
// produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size
@@ -27329,8 +27999,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
} else if (src0t == GGML_TYPE_Q3_K) {
GGML_ASSERT(false && "not implemented");
} else if (src0t == GGML_TYPE_Q5_K) {
size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};
size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};
@@ -27630,35 +28298,21 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
// the matmul kernels only write the rows an expert owns, so zero the skipped rows here
{
cl_int err;
cl_buffer_region region;
region.origin = offset2;
region.size = nb21 * ne21;
cl_mem buf_ids = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
region.origin = offsetd;
region.size = (size_t) ne0 * ne1 * ne2 * sizeof(float);
cl_mem buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const cl_uint ids_stride = nb21 / sizeof(int);
const cl_ulong nb1 = dst->nb[1];
const cl_ulong nb2 = dst->nb[2];
cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst;
CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &buf_ids));
CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_mem), &buf_dst));
CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_uint), &ne21));
CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_uint), &ne20));
CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_uint), &ids_stride));
CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_uint), &ne0));
CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &extra2->data_device));
CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_ulong), &offset2));
CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_ulong), &nb21));
CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_ulong), &nb1));
CL_CHECK(clSetKernelArg(kernel_zero, 6, sizeof(cl_ulong), &nb2));
CL_CHECK(clSetKernelArg(kernel_zero, 7, sizeof(int), &ne0));
size_t zero_local[3] = {1, 1, 64};
size_t zero_global[3] = {(size_t) ne21, (size_t) ne20, 64};
size_t zero_global[3] = {(size_t) ne20, (size_t) ne21, 64};
backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst);
CL_CHECK(clReleaseMemObject(buf_ids));
CL_CHECK(clReleaseMemObject(buf_dst));
}
int sgs = 32; // subgroup size
ggml/src/ggml-opencl/kernels/cvt.cl
+153
View File
@@ -2773,3 +2773,156 @@ kernel void kernel_restore_block_q2_k_noshuffle(
}
}
}
struct block_q3_K {
uchar hmask[QK_K/8];
uchar qs[QK_K/4];
uchar scales[12];
half d;
};
kernel void kernel_convert_block_q3_K(
global struct block_q3_K * src0,
global uchar * dst_qs,
global uchar * dst_hm,
global uchar * dst_sc,
global half * dst_d,
ulong n_blk
) {
const ulong i = get_global_id(0);
if (i >= n_blk) {
return;
}
global struct block_q3_K * b = src0 + i;
for (int j = 0; j < 64; ++j) {
dst_qs[64*i + j] = b->qs[j];
}
for (int j = 0; j < 32; ++j) {
dst_hm[32*i + j] = b->hmask[j];
}
for (int j = 0; j < 12; ++j) {
dst_sc[12*i + j] = b->scales[j];
}
dst_d[i] = b->d;
}
kernel void kernel_restore_block_q3_K(
global uchar * src_qs,
global uchar * src_hm,
global uchar * src_sc,
global half * src_d,
global struct block_q3_K * dst,
ulong n_blk
) {
const ulong i = get_global_id(0);
if (i >= n_blk) {
return;
}
global struct block_q3_K * b = dst + i;
for (int j = 0; j < 64; ++j) {
b->qs[j] = src_qs[64*i + j];
}
for (int j = 0; j < 32; ++j) {
b->hmask[j] = src_hm[32*i + j];
}
for (int j = 0; j < 12; ++j) {
b->scales[j] = src_sc[12*i + j];
}
b->d = src_d[i];
}
kernel void kernel_convert_block_q3_k_noshuffle(
global struct block_q3_K * src0,
global uchar * dst_qs, // QK_K/4 per block
global uchar * dst_hm, // QK_K/8 per block
global uint * dst_sc, // 3 per block
global half * dst_d, // 1
ulong n_blk
) {
const ulong i = get_global_id(0);
if (i >= n_blk) {
return;
}
global struct block_q3_K * b = (global struct block_q3_K *) src0 + i;
dst_d[i] = b->d;
for (int w = 0; w < 3; ++w) {
global const uchar * p = b->scales + 4*w;
dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24);
}
for (int g = 0; g < QK_K/4; ++g) {
uint pk = 0;
for (int t = 0; t < 4; ++t) {
const int e = 4*g + t;
const int sb = e >> 5;
const int ee = e & 31;
pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t));
}
dst_qs[(QK_K/4) * i + g] = (uchar)pk;
}
// two groups per hmask byte, so this loop owns the byte and cannot race
for (int t = 0; t < QK_K/8; ++t) {
uint v = 0;
for (int s = 0; s < 2; ++s) {
const int g = 2*t + s;
for (int u = 0; u < 4; ++u) {
const int e = 4*g + u;
const int sb = e >> 5;
const int ee = e & 31;
v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u));
}
}
dst_hm[(QK_K/8) * i + t] = (uchar)v;
}
}
kernel void kernel_restore_block_q3_k_noshuffle(
global uchar * src_qs,
global uchar * src_hm,
global uint * src_sc,
global half * src_d,
global struct block_q3_K * dst,
ulong n_blk
) {
const ulong i = get_global_id(0);
if (i >= n_blk) {
return;
}
global struct block_q3_K * b = (global struct block_q3_K *) dst + i;
b->d = src_d[i];
for (int w = 0; w < 3; ++w) {
const uint x = src_sc[3*i + w];
global uchar * p = b->scales + 4*w;
p[0] = (uchar)( x & 0xFF);
p[1] = (uchar)((x >> 8) & 0xFF);
p[2] = (uchar)((x >> 16) & 0xFF);
p[3] = (uchar)((x >> 24) & 0xFF);
}
for (int n2 = 0; n2 < 2; ++n2) {
for (int ee = 0; ee < 32; ++ee) {
uint v = 0;
for (int j = 0; j < 4; ++j) {
const int sb = 4*n2 + j;
const int e = 32*sb + ee;
v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j);
}
b->qs[32*n2 + ee] = (uchar)v;
}
}
for (int ee = 0; ee < 32; ++ee) {
uint v = 0;
for (int sb = 0; sb < 8; ++sb) {
const int e = 32*sb + ee;
const int g = e >> 2;
v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb;
}
b->hmask[ee] = (uchar)v;
}
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl
+92
View File
@@ -0,0 +1,92 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_qcom_reqd_sub_group_size
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
__attribute__((qcom_reqd_sub_group_size("full")))
#endif
kernel void kernel_gemm_noshuffle_q3_k_f32(
global const uchar * src0_qs,
global const uchar * src0_hm,
global const uint * src0_sc,
global const half * src0_d,
read_only image1d_buffer_t src1,
global float * dst,
ulong offsetd,
int m,
int n,
int k,
int n_no_padding
) {
dst = (global float *)((global char *)dst + offsetd);
const int gy = get_global_id(0);
const int row = get_global_id(1) * 4;
const int n_4 = n / 4;
if (row >= m) {
return;
}
half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
for (int i = 0; i < k; i += 16) {
const int is = (i / 16) & 15;
const int word = is >> 2;
const int shift = 8 * (is & 3);
const int sc_base = row + 3 * (i / 256) * m;
const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1));
const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word);
const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32);
const half4 d = vload4(0, src0_d + row + (i / 256) * m);
const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc));
for (int l = 0; l < 16; l += 4) {
const int ki = i + l;
const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m);
const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m);
#pragma unroll
for (int j = 0; j < 4; ++j) {
half8 b;
b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3));
const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale;
c0 += b * w.s0;
c1 += b * w.s1;
c2 += b * w.s2;
c3 += b * w.s3;
}
}
}
int idx = gy * 8 * m + row;
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
}
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl
+160
View File
@@ -0,0 +1,160 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define QK_K 256
#ifndef TILESIZE_N
#define TILESIZE_N 32
#endif
inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
const uint w = is >> 2;
const uint sh = 8u * (is & 3u);
const uint tb = (sc2 >> sh) & 0xFFu;
uint v;
if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); }
else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); }
else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
return (int)v - 32;
}
inline uint q3_k_pack(uint pk, uint hb) {
int v0 = (int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u);
int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u);
int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u);
int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u);
return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
| (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
}
// Pass staged operands by value; local dp4a operands miscompile on X2.
#define KQ_STAGE(p) vload4(0, (p))
inline int dot4_q8a_v(uint4 qw, uint4 a) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a(
__global const uchar * src0_qs,
__global const uchar * src0_hm,
__global const uint * src0_sc,
__global const half * src0_d,
__global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
__global const half * src1_da, // q8_1 per-block scale [N, K/32]
__global float * dst,
ulong offsetd,
int m, // output features (rows)
int n_no_padding, // tokens (cols)
int k // K (== ne00)
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const uint ib = sub >> 3; // super-block along K
const uint sb = sub & 7u;
const uint scb = rrow + (3u * ib) * (uint)m;
const uint sc0 = src0_sc[scb + 0u * (uint)m];
const uint sc1 = src0_sc[scb + 1u * (uint)m];
const uint sc2 = src0_sc[scb + 2u * (uint)m];
const float d_w = (float)src0_d[rrow + ib * (uint)m];
const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u);
const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u);
const uint qsb = rrow + (step >> 2) * (uint)m;
const uint hmb = rrow + (step >> 3) * (uint)m;
uint4 qlo, qhi;
{
const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m];
const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m];
const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m];
const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m];
qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m], h0 & 0xFu);
qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu);
qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m], h1 & 0xFu);
qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu);
qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m], h2 & 0xFu);
qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu);
qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m], h3 & 0xFu);
qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu);
}
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0])) \
+ dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1])))
rf.s0 = Q3_K_COL(b+0); rf.s1 = Q3_K_COL(b+1);
rf.s2 = Q3_K_COL(b+2); rf.s3 = Q3_K_COL(b+3);
#undef Q3_K_COL
acc[g] += LD4(sh_d, b) * rf;
}
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
}
#undef NGROUPS
}
ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl
+249
View File
@@ -0,0 +1,249 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define QK_K 256
#ifndef Q3_K_MV_NSG
#define Q3_K_MV_NSG 8
#endif
#ifndef Q3_K_MV_R
#define Q3_K_MV_R 2
#endif
inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
const uint w = is >> 2;
const uint sh = 8u * (is & 3u);
const uint tb = (sc2 >> sh) & 0xFFu;
uint v;
if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); }
else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); }
else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
return (int)v - 32;
}
inline float4 q3_k_vals(uint pk, uint hb) {
float4 v;
v.s0 = (float)((int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u));
v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u));
v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u));
v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u));
return v;
}
kernel void kernel_gemv_noshuffle_q3_k_f32(
global const uchar * src0_qs,
global const uchar * src0_hm,
global const uint * src0_sc,
global const half * src0_d,
global const float * src1,
ulong offset1,
global float * dst,
ulong offsetd,
int ne00, // K
int ne01, // M
int ne10, // activation row stride
int ne0 // dst row stride
) {
src1 = (global const float *)((global const char *)src1 + offset1);
dst = (global float *)((global char *)dst + offsetd);
const uint m = (uint)ne01;
const uint K = (uint)ne00;
const uint nsb = K / QK_K;
const uint lid = get_local_id(0);
const uint sgi = get_local_id(1);
const uint col = get_group_id(1);
global const float * y = src1 + (ulong)col * (uint)ne10;
#if Q3_K_MV_R == 4
const uint mq = m >> 2;
const uint j = get_group_id(0) * 64u + lid;
const uint row = j << 2;
float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f;
if (j < mq) {
global const uint * qsw = (global const uint *)src0_qs;
global const uint * hmw = (global const uint *)src0_hm;
for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
const half4 dh = vload4(j + ib * mq, src0_d);
const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc);
const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc);
const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc);
float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f;
for (uint sb = 0; sb < 8u; ++sb) {
const uint grp = ib * 64u + sb * 8u;
const uint qsb = j + grp * mq;
const uint hmb = j + (grp >> 1) * mq;
for (uint h = 0; h < 2u; ++h) { // two 16-weight halves
const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h);
const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h);
float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
for (uint u = 0; u < 4u; ++u) {
const uint gg = 4u*h + u;
const uint qsv = qsw[qsb + gg * mq]; // four rows, one load
const uint hmv = hmw[hmb + (gg >> 1) * mq];
const uint hsh = 4u * (gg & 1u);
const float4 yv = vload4(grp + gg, y); // shared by all four
a0 += dot(yv, q3_k_vals((qsv ) & 0xFFu, ((hmv ) >> hsh) & 0xFu));
a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu));
a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu));
a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu));
}
acc0 += (float)l0 * a0;
acc1 += (float)l1 * a1;
acc2 += (float)l2 * a2;
acc3 += (float)l3 * a3;
}
}
sumf += (float)dh.s0 * acc0;
sumf1 += (float)dh.s1 * acc1;
sumf2 += (float)dh.s2 * acc2;
sumf3 += (float)dh.s3 * acc3;
}
}
#elif Q3_K_MV_R == 2
const uint mh = m >> 1;
const uint j = get_group_id(0) * 64u + lid;
const uint row = j << 1;
float sumf = 0.f;
float sumf1 = 0.f;
if (j < mh) {
global const ushort * qsu = (global const ushort *)src0_qs;
global const ushort * hmu = (global const ushort *)src0_hm;
for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
const half2 dh = vload2(j + ib * mh, src0_d);
const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc);
const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc);
const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc);
float acc0 = 0.f, acc1 = 0.f;
for (uint sb = 0; sb < 8u; ++sb) {
const uint grp = ib * 64u + sb * 8u;
const uint qsb = j + grp * mh;
const uint hmb = j + (grp >> 1) * mh;
for (uint h = 0; h < 2u; ++h) { // two 16-weight halves
const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
float a0 = 0.f, a1 = 0.f;
for (uint u = 0; u < 4u; ++u) {
const uint gg = 4u*h + u;
const uint qsv = (uint)qsu[qsb + gg * mh];
const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh];
const uint hsh = 4u * (gg & 1u);
const float4 yv = vload4(grp + gg, y);
a0 += dot(yv, q3_k_vals( qsv & 0xFFu, ( hmv >> hsh) & 0xFu));
a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu));
}
acc0 += (float)l0 * a0;
acc1 += (float)l1 * a1;
}
}
sumf += (float)dh.s0 * acc0;
sumf1 += (float)dh.s1 * acc1;
}
}
#else
const uint row = get_group_id(0) * 64u + lid;
float sumf = 0.f;
if (row < m) {
for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
const float d = (float)src0_d[row + ib * m];
const uint s0 = src0_sc[row + (3u * ib + 0u) * m];
const uint s1 = src0_sc[row + (3u * ib + 1u) * m];
const uint s2 = src0_sc[row + (3u * ib + 2u) * m];
float acc = 0.f;
for (uint sb = 0; sb < 8u; ++sb) {
const uint grp = ib * 64u + sb * 8u;
const uint qsb = row + grp * m;
const uint hmb = row + (grp >> 1) * m;
for (uint h = 0; h < 2u; ++h) {
const int l = q3_k_scale(s0, s1, s2, 2u*sb + h);
float a = 0.f;
for (uint u = 0; u < 4u; ++u) {
const uint gg = 4u*h + u;
const uint qsv = (uint)src0_qs[qsb + gg * m];
const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m];
const float4 yv = vload4(grp + gg, y);
a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu));
}
acc += (float)l * a;
}
}
sumf += d * acc;
}
}
#endif
#if Q3_K_MV_NSG > 1
#if Q3_K_MV_R == 4
__local float4 part[Q3_K_MV_NSG][64];
part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3);
barrier(CLK_LOCAL_MEM_FENCE);
if (sgi != 0) {
return;
}
for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
const float4 p = part[s][lid];
sumf += p.s0;
sumf1 += p.s1;
sumf2 += p.s2;
sumf3 += p.s3;
}
#elif Q3_K_MV_R == 2
__local float2 part[Q3_K_MV_NSG][64];
part[sgi][lid] = (float2)(sumf, sumf1);
barrier(CLK_LOCAL_MEM_FENCE);
if (sgi != 0) {
return;
}
for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
const float2 p = part[s][lid];
sumf += p.s0;
sumf1 += p.s1;
}
#else
__local float part[Q3_K_MV_NSG][64];
part[sgi][lid] = sumf;
barrier(CLK_LOCAL_MEM_FENCE);
if (sgi != 0) {
return;
}
for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
sumf += part[s][lid];
}
#endif
#endif
#if Q3_K_MV_R == 4
if (j < mq) {
vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row);
}
#elif Q3_K_MV_R == 2
if (j < mh) {
vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
}
#else
if (row < m) {
dst[(ulong)col * (uint)ne0 + row] = sumf;
}
#endif
}
ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl
+8 -4
View File
@@ -57,13 +57,17 @@ kernel void kernel_add_id_add_id_swiglu_oai(
global const float * g_row = (global const float *)(src_g + i1*nb01_g + i2*nb02_g);
global const float * u_row = (global const float *)(src_u + i1*nb01_u + i2*nb02_u);
global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
global float * d_row = (global float *)(dst + i1*nbd1 + i2*nbd2);
for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
float x0 = g_row[i0] + gb_row[i0];
float x1 = u_row[i0] + ub_row[i0];
float x0 = g_row[i0];
float x1 = u_row[i0];
if (i11 != -1) {
global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
x0 += gb_row[i0];
x1 += ub_row[i0];
}
x0 = min(x0, limit);
x1 = max(min(x1, limit), -limit);
ggml/src/ggml-opencl/kernels/moe_combine.cl
+5 -2
View File
@@ -42,8 +42,11 @@ __kernel void kernel_moe_combine_bias_f32(
float4 acc = (float4)(0.0f);
for (int e = 0; e < k; ++e) {
const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int)));
__global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow);
float4 v = vload4(0, E + (uint)e*e1);
if (i11 != -1) {
__global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
v += vload4(0, Brow);
}
acc = mad(v, (float4)(W[(uint)e*w1]), acc);
}
ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl
-26
View File
@@ -80,32 +80,6 @@ __kernel void kernel_moe_scatter(
emap[tile_idx] = val;
}
// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here
__kernel void kernel_moe_zero_dst(
__global const int * input,
__global float * dst,
uint N,
uint topK,
uint n_experts,
uint ne0
) {
uint n = get_global_id(0);
uint k = get_global_id(1);
if (n >= N || k >= topK) {
return;
}
if (input[n * n_experts + k] != -1) {
return;
}
__global float * dst_row = dst + ((ulong)n * topK + k) * ne0;
for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) {
dst_row[i] = 0.0f;
}
}
// Deterministic replacement for kernel_moe_scatter.
//
// kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]),
ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl
+22
View File
@@ -0,0 +1,22 @@
kernel void kernel_moe_zero_dst(
global const char * ids,
ulong offset_ids,
ulong nb_ids1,
global char * dst,
ulong offset_dst,
ulong nb1,
ulong nb2,
int ne0
) {
const size_t i1 = get_group_id(0);
const size_t i2 = get_group_id(1);
const int expert = *((global const int *)(ids + offset_ids + i1*sizeof(int) + i2*nb_ids1));
if (expert != -1) {
return;
}
global float * row = (global float *)(dst + offset_dst + i1*nb1 + i2*nb2);
for (size_t i0 = get_local_id(2); i0 < (size_t)ne0; i0 += get_local_size(2)) {
row[i0] = 0.0f;
}
}
ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl
+219
View File
@@ -0,0 +1,219 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#define INTEL_GPU 1
#endif
#define QK_K 256
struct block_q3_K {
uchar hmask[QK_K/8];
uchar qs[QK_K/4];
uchar scales[12];
half d;
};
#define LOAD_VEC_A 4
#define LOAD_VEC_B 4
#define BM 64
#define BN 64
#ifndef BK
#define BK 32
#endif
#ifdef INTEL_GPU
#define TM 8
#else
#define TM 4
#endif
#define TN 8
#ifndef LM_HALF
#define LM_HALF 0
#endif
#if LM_HALF
typedef half lm_st;
#define LM_LD4(p, i) convert_float4(vload4((i), (p)))
#else
typedef float lm_st;
#define LM_LD4(p, i) vload4((i), (p))
#endif
kernel void kernel_mul_mm_q3_k_f32_l4_lm(
global uchar * src0_qs,
global uchar * src0_hm,
global uchar * src0_sc,
global half * src0_d,
ulong offset0,
global float4 * src1,
ulong offset1,
global float * dst,
ulong offsetd,
int ne00,
int ne01,
int ne02,
int ne11,
int ne12,
int stride_a,
int stride_b,
int stride_d,
int batch_stride_a,
int batch_stride_b,
int batch_stride_d,
int r2,
int r3
) {
src1 = (global float4*)((global char*)src1 + offset1);
dst = (global float *)((global char*)dst + offsetd);
local lm_st buf_a[BM * BK];
local lm_st buf_b[BN * BK];
const int batch_idx = get_global_id(2);
const int i13 = batch_idx / ne12;
const int i12 = batch_idx % ne12;
const int i03 = i13 / r3;
const int i02 = i12 / r2;
const int batch_idx_a = i03 * ne02 + i02;
const int ir = get_group_id(0);
const int ic = get_group_id(1);
const int tid = get_local_id(0);
const int th_r = tid % (BM / TM);
const int th_c = tid / (BM / TM);
const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
// pos_a counts elements, not blocks - one q3_K super block holds QK_K of them
int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
float4 sums4[(TM/4) * TN];
float4 cache_a4[TM/4];
for (int i = 0; i < (TM/4) * TN; i++) {
sums4[i] = (float4)(0.0f);
}
for (int block = 0; block < ne00; block += BK) {
for (int l = 0; l < BM; l += loadstride_a) {
if (ir*BM + loadc_a + l < ne01) {
int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
int ib = idx / QK_K;
ib += offset0/sizeof(struct block_q3_K);
int e = idx % QK_K;
// Only qs advances in the second half of a super-block.
int n = e >> 7;
int rem = e & 127;
int j = rem >> 5;
int s = (rem >> 4) & 1;
int l16 = rem & 15;
int is = 8*n + 2*j + s;
int shift = 2*j;
uchar m = (uchar)(1u << (4*n + j));
global ushort * a = (global ushort *)(src0_sc + 12*ib);
uint a0 = (uint)a[0] | ((uint)a[1] << 16);
uint a1 = (uint)a[2] | ((uint)a[3] << 16);
uint tmp = (uint)a[4] | ((uint)a[5] << 16);
uint g = is >> 2;
uint w = (g & 1u) ? a1 : a0;
w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4);
uint sc6 = (w >> (8*(is & 3))) & 0xFFu;
float dl = (float)src0_d[ib] * ((float)sc6 - 32.0f);
uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16);
uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16);
float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3,
(q.s1 >> shift) & 3,
(q.s2 >> shift) & 3,
(q.s3 >> shift) & 3));
float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f,
(hb.s1 & m) ? 0.f : 4.f,
(hb.s2 & m) ? 0.f : 4.f,
(hb.s3 & m) ? 0.f : 4.f);
float4 v1 = (qv - hv) * dl;
buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
} else {
buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
}
}
for (int l = 0; l < BN; l += loadstride_b) {
if (ic*BN + loadc_b + l < ne11) {
int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
} else {
buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
}
}
barrier(CLK_LOCAL_MEM_FENCE);
pos_a += BK;
pos_b += BK / LOAD_VEC_B;
for (int i = 0; i < BK; i++) {
for (int a = 0; a < TM/4; a++) {
cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a);
}
for (int cc = 0; cc < TN; cc++) {
const float cache_b = buf_b[(i) * BN + th_c * TN + cc];
for (int a = 0; a < TM/4; a++) {
const int sums_idx = cc*(TM/4) + a;
sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]);
}
}
}
barrier(CLK_LOCAL_MEM_FENCE);
}
const int dr = ir * BM + th_r * TM;
const int dc = ic * BN + th_c * TN;
const ulong offsets = batch_idx * batch_stride_d;
for (int cc = 0; cc < TN; cc++) {
for (int a = 0; a < TM/4; a++) {
const float4 v = sums4[cc * (TM/4) + a];
const float vs[4] = { v.s0, v.s1, v.s2, v.s3 };
for (int k = 0; k < 4; k++) {
if (dr + 4*a + k < ne01 && dc + cc < ne11) {
dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k];
}
}
}
}
}
ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl
+197
View File
@@ -0,0 +1,197 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
#define INTEL_GPU 1
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
#elif defined(cl_qcom_reqd_sub_group_size)
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
#endif
//------------------------------------------------------------------------------
// block_q3_K
//------------------------------------------------------------------------------
#define QK_K 256
// 16 blocks of 16 elements each
// weight is represented as x = a * q
typedef struct {
uchar hmask[QK_K/8]; // quants - high bit
uchar qs[QK_K/4]; // quants - low 2 bits
uchar scales[12]; // scales, quantized with 6 bits
half d; // super-block scale
} block_q3_K;
#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
#define N_DST 2 // number of rows each SIMD group works on
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // SIMD group size
#elif defined (ADRENO_GPU)
#define N_DST 2
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
#undef BLOCK_STRIDE
// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
#define BLOCK_STRIDE (N_SIMDWIDTH/8)
#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mv_q3_K_f32(
global char * src0,
ulong offset0,
global char * src1,
ulong offset1,
global char * dst,
ulong offsetd,
int ne00,
int ne01,
ulong nb01,
ulong nb02,
ulong nb03,
int ne12,
ulong nb11,
ulong nb12,
ulong nb13,
int ne0,
int ne1,
int r2,
int r3
) {
src0 = src0 + offset0;
src1 = src1 + offset1;
dst = dst + offsetd;
int nb = ne00/QK_K;
int r0 = get_group_id(0);
int r1 = get_group_id(1);
int im = get_group_id(2);
int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
int i12 = im%ne12;
int i13 = im/ne12;
ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
global block_q3_K * x = (global block_q3_K *) (src0 + offset_src0);
global float * yy = (global float *) (src1 + offset_src1);
float yl[32];
// Eight lanes cover one super-block.
int ix = get_sub_group_local_id() % BLOCK_STRIDE;
int tid = get_sub_group_local_id() / BLOCK_STRIDE;
int ip = tid/4; // 0 or 1
int il = 2*((tid%4)/2); // 0 or 2
int ir = tid%2;
int l0 = 8*ir;
// Shift the masks instead of indexing private arrays to avoid spills.
ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2));
int shift = 2*il;
float v1 = il == 0 ? 4.f : 64.f;
float v2 = 4.f * v1;
int s_shift1 = 4*ip;
int s_shift2 = s_shift1 + il;
int q_offset = 32*ip + l0;
int y_offset = 128*ip + 32*il + l0;
global float * y1 = yy + ix*QK_K + y_offset;
float sumf1[N_DST] = {0.f};
float sumf2[N_DST] = {0.f};
float all_sum;
for (int i = ix; i < nb; i += BLOCK_STRIDE) {
for (int l = 0; l < 8; ++l) {
yl[l+ 0] = y1[l+ 0];
yl[l+ 8] = y1[l+16];
yl[l+16] = y1[l+32];
yl[l+24] = y1[l+48];
}
global ushort * q = (global ushort *)(x[i].qs + q_offset);
global ushort * h = (global ushort *)(x[i].hmask + l0);
global ushort * a = (global ushort *)(x[i].scales);
global half * dh = &x[i].d;
// only ne01 output rows exist; reading past them can pull in an inf scale
for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
float d_all = dh[0];
uint s32 = (uint)a[4] | ((uint)a[5] << 16);
uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
char4 sc = as_char4(s32);
float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
for (int l = 0; l < 8; l += 2) {
int qs = q[l/2];
s1 += yl[l+0] * (qs & qm.s0);
s2 += yl[l+1] * (qs & qm.s1);
s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
s4 += yl[l+16] * (qs & qm.s2);
s5 += yl[l+17] * (qs & qm.s3);
s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
}
float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
sumf1[row] += d1 * (sc.s0 - 32);
sumf2[row] += d2 * (sc.s2 - 32);
s1 = s2 = s3 = s4 = s5 = s6 = 0;
for (int l = 0; l < 8; l += 2) {
int qs = q[l/2+8];
s1 += yl[l+8] * (qs & qm.s0);
s2 += yl[l+9] * (qs & qm.s1);
s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
s4 += yl[l+24] * (qs & qm.s2);
s5 += yl[l+25] * (qs & qm.s3);
s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
}
d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
sumf1[row] += d1 * (sc.s1 - 32);
sumf2[row] += d2 * (sc.s3 - 32);
q += nb01/2;
h += nb01/2;
a += nb01/2;
dh += nb01/2;
}
y1 += BLOCK_STRIDE * QK_K;
}
global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
for (int row = 0; row < N_DST; ++row) {
all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
if (first_row + row < ne01) {
if (get_sub_group_local_id() == 0) {
dst_f32[first_row + row] = all_sum;
}
}
}
}
ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl
+199
View File
@@ -0,0 +1,199 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
#define INTEL_GPU 1
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
#elif defined(cl_qcom_reqd_sub_group_size)
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
#endif
//------------------------------------------------------------------------------
// block_q3_K
//------------------------------------------------------------------------------
#define QK_K 256
// 16 blocks of 16 elements each
// weight is represented as x = a * q
typedef struct {
uchar hmask[QK_K/8]; // quants - high bit
uchar qs[QK_K/4]; // quants - low 2 bits
uchar scales[12]; // scales, quantized with 6 bits
half d; // super-block scale
} block_q3_K;
#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
#define N_DST 2 // number of rows each SIMD group works on
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // SIMD group size
#elif defined (ADRENO_GPU)
#define N_DST 2
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
#undef BLOCK_STRIDE
// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
#define BLOCK_STRIDE (N_SIMDWIDTH/8)
#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mv_q3_K_f32_flat(
global uchar * src0_qs,
global uchar * src0_hm,
global uchar * src0_sc,
global half * src0_d,
ulong offset0,
global char * src1,
ulong offset1,
global char * dst,
ulong offsetd,
int ne00,
int ne01,
ulong nb01,
ulong nb02,
ulong nb03,
int ne12,
ulong nb11,
ulong nb12,
ulong nb13,
int ne0,
int ne1,
int r2,
int r3
) {
src1 = src1 + offset1;
dst = dst + offsetd;
int nb = ne00/QK_K;
int r0 = get_group_id(0);
int r1 = get_group_id(1);
int im = get_group_id(2);
int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
int i12 = im%ne12;
int i13 = im/ne12;
ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K);
ulong row_stride = nb01/sizeof(block_q3_K);
ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
global float * yy = (global float *) (src1 + offset_src1);
float yl[32];
// Eight lanes cover one super-block.
int ix = get_sub_group_local_id() % BLOCK_STRIDE;
int tid = get_sub_group_local_id() / BLOCK_STRIDE;
int ip = tid/4; // 0 or 1
int il = 2*((tid%4)/2); // 0 or 2
int ir = tid%2;
int l0 = 8*ir;
// Shift the masks instead of indexing private arrays to avoid spills.
ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2));
int shift = 2*il;
float v1 = il == 0 ? 4.f : 64.f;
float v2 = 4.f * v1;
int s_shift1 = 4*ip;
int s_shift2 = s_shift1 + il;
int q_offset = 32*ip + l0;
int y_offset = 128*ip + 32*il + l0;
global float * y1 = yy + ix*QK_K + y_offset;
float sumf1[N_DST] = {0.f};
float sumf2[N_DST] = {0.f};
float all_sum;
for (int i = ix; i < nb; i += BLOCK_STRIDE) {
for (int l = 0; l < 8; ++l) {
yl[l+ 0] = y1[l+ 0];
yl[l+ 8] = y1[l+16];
yl[l+16] = y1[l+32];
yl[l+24] = y1[l+48];
}
global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset);
global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0);
global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i));
global half * dh = src0_d + offset_src0 + i;
// only ne01 output rows exist; reading past them can pull in an inf scale
for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
float d_all = dh[0];
uint s32 = (uint)a[4] | ((uint)a[5] << 16);
uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
char4 sc = as_char4(s32);
float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
for (int l = 0; l < 8; l += 2) {
int qs = q[l/2];
s1 += yl[l+0] * (qs & qm.s0);
s2 += yl[l+1] * (qs & qm.s1);
s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
s4 += yl[l+16] * (qs & qm.s2);
s5 += yl[l+17] * (qs & qm.s3);
s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
}
float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
sumf1[row] += d1 * (sc.s0 - 32);
sumf2[row] += d2 * (sc.s2 - 32);
s1 = s2 = s3 = s4 = s5 = s6 = 0;
for (int l = 0; l < 8; l += 2) {
int qs = q[l/2+8];
s1 += yl[l+8] * (qs & qm.s0);
s2 += yl[l+9] * (qs & qm.s1);
s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
s4 += yl[l+24] * (qs & qm.s2);
s5 += yl[l+25] * (qs & qm.s3);
s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
}
d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
sumf1[row] += d1 * (sc.s1 - 32);
sumf2[row] += d2 * (sc.s3 - 32);
q += row_stride*32;
h += row_stride*16;
a += row_stride*6;
dh += row_stride*1;
}
y1 += BLOCK_STRIDE * QK_K;
}
global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
for (int row = 0; row < N_DST; ++row) {
all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
if (first_row + row < ne01) {
if (get_sub_group_local_id() == 0) {
dst_f32[first_row + row] = all_sum;
}
}
}
}
ggml/src/ggml-sycl/concat.cpp
+22 -232
View File
@@ -196,7 +196,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
}
}
static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
template <typename block_t, int qk, ggml_type qtype>
static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
@@ -204,25 +205,25 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *
const int32_t dim = ((int32_t *) dst->op_params)[0];
GGML_ASSERT(src0->type == GGML_TYPE_Q4_0);
GGML_ASSERT(src1->type == GGML_TYPE_Q4_0);
GGML_ASSERT(dst->type == GGML_TYPE_Q4_0);
GGML_ASSERT(src0->ne[0] % QK4_0 == 0);
GGML_ASSERT(src1->ne[0] % QK4_0 == 0);
GGML_ASSERT(dst->ne[0] % QK4_0 == 0);
GGML_ASSERT(src0->type == qtype);
GGML_ASSERT(src1->type == qtype);
GGML_ASSERT(dst->type == qtype);
GGML_ASSERT(src0->ne[0] % qk == 0);
GGML_ASSERT(src1->ne[0] % qk == 0);
GGML_ASSERT(dst->ne[0] % qk == 0);
const int ne00_blk = src0->ne[0] / QK4_0;
const int ne0_blk = dst->ne[0] / QK4_0;
const int ne00_blk = src0->ne[0] / qk;
const int ne0_blk = dst->ne[0] / qk;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
const block_q4_0 * src0_d = (const block_q4_0 *) src0->data;
const block_q4_0 * src1_d = (const block_q4_0 *) src1->data;
block_q4_0 * dst_d = (block_q4_0 *) dst->data;
const size_t type_size = sizeof(block_q4_0);
const block_t * src0_d = (const block_t *) src0->data;
const block_t * src1_d = (const block_t *) src1->data;
block_t * dst_d = (block_t *) dst->data;
const size_t type_size = sizeof(block_t);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
concat_T_sycl<block_q4_0>(
concat_T_sycl<block_t>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
@@ -237,223 +238,12 @@ static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
concat_T_sycl_non_cont<block_q4_0>(
concat_T_sycl_non_cont<block_t>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
}
}
static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
queue_ptr stream = ctx.stream();
const int32_t dim = ((int32_t *) dst->op_params)[0];
GGML_ASSERT(src0->type == GGML_TYPE_Q4_1);
GGML_ASSERT(src1->type == GGML_TYPE_Q4_1);
GGML_ASSERT(dst->type == GGML_TYPE_Q4_1);
GGML_ASSERT(src0->ne[0] % QK4_1 == 0);
GGML_ASSERT(src1->ne[0] % QK4_1 == 0);
GGML_ASSERT(dst->ne[0] % QK4_1 == 0);
const int ne00_blk = src0->ne[0] / QK4_1;
const int ne0_blk = dst->ne[0] / QK4_1;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
const block_q4_1 * src0_d = (const block_q4_1 *) src0->data;
const block_q4_1 * src1_d = (const block_q4_1 *) src1->data;
block_q4_1 * dst_d = (block_q4_1 *) dst->data;
const size_t type_size = sizeof(block_q4_1);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
concat_T_sycl<block_q4_1>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
dst->ne[1], dst->ne[2], dim, stream);
}
} else {
const size_t size0 = ggml_nbytes(src0);
const size_t size1 = ggml_nbytes(src1);
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
concat_T_sycl_non_cont<block_q4_1>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
}
}
static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
queue_ptr stream = ctx.stream();
const int32_t dim = ((int32_t *) dst->op_params)[0];
GGML_ASSERT(src0->type == GGML_TYPE_Q5_0);
GGML_ASSERT(src1->type == GGML_TYPE_Q5_0);
GGML_ASSERT(dst->type == GGML_TYPE_Q5_0);
GGML_ASSERT(src0->ne[0] % QK5_0 == 0);
GGML_ASSERT(src1->ne[0] % QK5_0 == 0);
GGML_ASSERT(dst->ne[0] % QK5_0 == 0);
const int ne00_blk = src0->ne[0] / QK5_0;
const int ne0_blk = dst->ne[0] / QK5_0;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
const block_q5_0 * src0_d = (const block_q5_0 *) src0->data;
const block_q5_0 * src1_d = (const block_q5_0 *) src1->data;
block_q5_0 * dst_d = (block_q5_0 *) dst->data;
const size_t type_size = sizeof(block_q5_0);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
concat_T_sycl<block_q5_0>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
dst->ne[1], dst->ne[2], dim, stream);
}
} else {
const size_t size0 = ggml_nbytes(src0);
const size_t size1 = ggml_nbytes(src1);
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
concat_T_sycl_non_cont<block_q5_0>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
}
}
static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
queue_ptr stream = ctx.stream();
const int32_t dim = ((int32_t *) dst->op_params)[0];
GGML_ASSERT(src0->type == GGML_TYPE_Q5_1);
GGML_ASSERT(src1->type == GGML_TYPE_Q5_1);
GGML_ASSERT(dst->type == GGML_TYPE_Q5_1);
GGML_ASSERT(src0->ne[0] % QK5_1 == 0);
GGML_ASSERT(src1->ne[0] % QK5_1 == 0);
GGML_ASSERT(dst->ne[0] % QK5_1 == 0);
const int ne00_blk = src0->ne[0] / QK5_1;
const int ne0_blk = dst->ne[0] / QK5_1;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
const block_q5_1 * src0_d = (const block_q5_1 *) src0->data;
const block_q5_1 * src1_d = (const block_q5_1 *) src1->data;
block_q5_1 * dst_d = (block_q5_1 *) dst->data;
const size_t type_size = sizeof(block_q5_1);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
concat_T_sycl<block_q5_1>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
dst->ne[1], dst->ne[2], dim, stream);
}
} else {
const size_t size0 = ggml_nbytes(src0);
const size_t size1 = ggml_nbytes(src1);
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
concat_T_sycl_non_cont<block_q5_1>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
}
}
static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
queue_ptr stream = ctx.stream();
const int32_t dim = ((int32_t *) dst->op_params)[0];
GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);
GGML_ASSERT(src1->type == GGML_TYPE_Q8_0);
GGML_ASSERT(dst->type == GGML_TYPE_Q8_0);
GGML_ASSERT(src0->ne[0] % QK8_0 == 0);
GGML_ASSERT(src1->ne[0] % QK8_0 == 0);
GGML_ASSERT(dst->ne[0] % QK8_0 == 0);
const int ne00_blk = src0->ne[0] / QK8_0;
const int ne0_blk = dst->ne[0] / QK8_0;
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
const block_q8_0 * src0_d = (const block_q8_0 *) src0->data;
const block_q8_0 * src1_d = (const block_q8_0 *) src1->data;
block_q8_0 * dst_d = (block_q8_0 *) dst->data;
const size_t type_size = sizeof(block_q8_0);
if (dim != 3) {
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
concat_T_sycl<block_q8_0>(
src0_d + i3 * (src0->nb[3] / type_size),
src1_d + i3 * (src1->nb[3] / type_size),
dst_d + i3 * (dst->nb[3] / type_size),
ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
dst->ne[1], dst->ne[2], dim, stream);
}
} else {
const size_t size0 = ggml_nbytes(src0);
const size_t size1 = ggml_nbytes(src1);
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
}
} else {
concat_T_sycl_non_cont<block_q8_0>(
stream, (const char *) src0->data, (const char *) src1->data,
(char *) dst->data,
ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3],
src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3],
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
@@ -487,19 +277,19 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
concat_impl_sycl<int8_t>(ctx, dst);
break;
case GGML_TYPE_Q4_0:
concat_impl_q4_0_sycl(ctx, dst);
concat_impl_q_sycl<block_q4_0, QK4_0, GGML_TYPE_Q4_0>(ctx, dst);
break;
case GGML_TYPE_Q4_1:
concat_impl_q4_1_sycl(ctx, dst);
concat_impl_q_sycl<block_q4_1, QK4_1, GGML_TYPE_Q4_1>(ctx, dst);
break;
case GGML_TYPE_Q5_0:
concat_impl_q5_0_sycl(ctx, dst);
concat_impl_q_sycl<block_q5_0, QK5_0, GGML_TYPE_Q5_0>(ctx, dst);
break;
case GGML_TYPE_Q5_1:
concat_impl_q5_1_sycl(ctx, dst);
concat_impl_q_sycl<block_q5_1, QK5_1, GGML_TYPE_Q5_1>(ctx, dst);
break;
case GGML_TYPE_Q8_0:
concat_impl_q8_0_sycl(ctx, dst);
concat_impl_q_sycl<block_q8_0, QK8_0, GGML_TYPE_Q8_0>(ctx, dst);
break;
default:
fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type));
ggml/src/ggml-sycl/ggml-sycl.cpp
+367 -490
View File
File diff suppressed because it is too large. Load diff
ggml/src/ggml-sycl/mmvq.cpp
+160 -670
View File
File diff suppressed because it is too large. Load diff
ggml/src/ggml-vulkan/ggml-vulkan-types.h
+6 -6
View File
@@ -848,18 +848,18 @@ struct vk_device_struct {
vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
vk_pipeline pipeline_norm_f32;
vk_pipeline pipeline_group_norm_f32;
vk_pipeline pipeline_rms_norm_f32;
vk_pipeline pipeline_rms_norm_mul_f32;
vk_pipeline pipeline_rms_norm_mul_add_f32;
vk_pipeline pipeline_rms_norm_mul_add_mul_f32;
std::array<vk_pipeline, 3> pipeline_rms_norm_f32;
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_f32;
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_f32;
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_mul_f32;
vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
vk_pipeline pipeline_rms_norm_set_rows_f32_f16;
vk_pipeline pipeline_rms_norm_partials_f32;
vk_pipeline pipeline_rms_norm_mul_partials_f32;
vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;
vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f32;
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f16;
vk_pipeline pipeline_rms_norm_back_f32;
vk_pipeline pipeline_l2_norm_f32;
ggml/src/ggml-vulkan/ggml-vulkan.cpp
+36 -12
View File
@@ -3256,21 +3256,34 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_f32, "rms_norm_mul_add_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_f32, "rms_norm_mul_add_mul_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support;
const uint32_t rms_set_rows_block_size = 128;
#define RMS_SHADER(name) (rms_subgroups ? name##_subgroup_len : name##_len), (rms_subgroups ? name##_subgroup_data : name##_data)
// 128/256/512 block sizes, picked by row length
for (uint32_t i = 0; i < 3; ++i) {
const uint32_t block_size = 128u << i;
const std::string suffix = "_" + std::to_string(block_size);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size}, 1, true, rms_subgroups);
}
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", rms_norm_set_rows_f32_f32_len, rms_norm_set_rows_f32_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", rms_norm_set_rows_f32_f16_len, rms_norm_set_rows_f32_f16_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", RMS_SHADER(rms_norm_set_rows_f32_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", RMS_SHADER(rms_norm_set_rows_f32_f16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true);
if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) {
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true);
for (uint32_t i = 0; i < 3; ++i) {
const uint32_t block_size = 128u << i;
const std::string suffix = "_" + std::to_string(block_size);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f32[i], "rms_norm_mul_rope_f32_f32" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f32), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f16[i], "rms_norm_mul_rope_f32_f16" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f16), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
}
}
#undef RMS_SHADER
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);
@@ -8740,6 +8753,16 @@ static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx,
return ggml_vk_conv_select_shape(ctx, K, NPQ);
}
static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
if (pipelines[0] && ncols <= 128) {
return pipelines[0];
}
if (pipelines[1] && ncols <= 256) {
return pipelines[1];
}
return pipelines[2];
}
static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) {
switch (op) {
case GGML_OP_GET_ROWS:
@@ -8980,7 +9003,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
if (ctx->do_add_rms_partials) {
return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32;
}
return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32;
return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]);
}
return nullptr;
case GGML_OP_RMS_NORM_BACK:
@@ -10942,8 +10965,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
pipeline = do_post_multiply ?
ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32;
} else {
pipeline = do_post_multiply ?
ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32;
pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ?
ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]);
}
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
if (ctx->do_add_rms_partials) {
@@ -11016,7 +11039,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
pc.bin = bin;
pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride);
vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32;
const auto & pipelines = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32;
vk_pipeline pipeline = ggml_vk_rms_norm_pipeline(pipelines, src0->ne[0]);
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp
+41 -15
View File
@@ -3,6 +3,15 @@
#include "generic_binary_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
#if USE_SUBGROUP_ADD
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#endif
layout (constant_id = 3) const int BLOCK_SIZE = 512;
layout(local_size_x_id = 3, local_size_y = 1, local_size_z = 1) in;
#if RMS_NORM_ROPE_FUSION
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
@@ -11,7 +20,8 @@ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
// data is passed from rms_norm -> rope through shared memory.
// rms_norm calls this data_d, rope calls this rope_data_a.
// Binding 2 is not used
shared FLOAT_TYPE rope_data_a[1024];
// host picks a block size such that ncols <= 2*BLOCK_SIZE
shared FLOAT_TYPE rope_data_a[2*BLOCK_SIZE];
#define data_d rope_data_a
layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];};
@@ -38,16 +48,11 @@ layout (binding = 3) readonly buffer I {uvec2 data_i[];};
#endif
#extension GL_EXT_control_flow_attributes : enable
#define BLOCK_SIZE 512
layout (constant_id = 1) const bool do_multiply = false;
#if RMS_NORM_ADD_FUSION
layout (constant_id = 2) const bool do_post_multiply = false;
#endif
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
shared FLOAT_TYPE sumsh[BLOCK_SIZE];
void rms_norm(uint num_iters) {
@@ -88,17 +93,38 @@ void rms_norm(uint num_iters) {
sum += xi * xi;
}
sumsh[tid] = sum;
// sum up partial sums and write back result
barrier();
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
if (tid < s) {
sum += sumsh[tid + s];
sumsh[tid] = sum;
#if USE_SUBGROUP_ADD
sum = subgroupAdd(sum);
if (subgroupElect()) {
sumsh[gl_SubgroupID] = sum;
}
barrier();
}
sum = sumsh[0];
sum = FLOAT_TYPE(0.0f);
if(gl_NumSubgroups <= 4) { // fast path to skip the 2nd subgroupAdd
[[unroll]] for(uint i=0; i < gl_NumSubgroups; ++i) {
sum += sumsh[i];
}
}
else {
// Each subgroup reads the partials, so one workgroup barrier is enough.
for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) {
sum += sumsh[i];
}
sum = subgroupAdd(sum);
}
#else
sumsh[tid] = sum;
// sum up partial sums and write back result
barrier();
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
if (tid < s) {
sum += sumsh[tid + s];
sumsh[tid] = sum;
}
barrier();
}
sum = sumsh[0];
#endif
const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));
ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+10 -6
View File
@@ -853,14 +853,18 @@ void process_shaders() {
// Norms
string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("rms_norm_mul_add_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}}));
for (bool use_subgroups : {false, true}) {
const std::string suffix = use_subgroups ? "_subgroup" : "";
const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict);
string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}}));
string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE'
string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
string_to_spv("rms_norm_mul_rope_f32_f16" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
}
string_to_spv("rms_norm_mul_add_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}}));
string_to_spv("rms_norm_set_rows_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}));
string_to_spv("rms_norm_set_rows_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}));
string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("rms_norm_mul_rope_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
string_to_spv("rms_norm_mul_rope_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
ggml/src/ggml.c
+1 -1
View File
@@ -3503,7 +3503,7 @@ static struct ggml_tensor * ggml_scale_impl(
float s,
float b,
bool inplace) {
GGML_ASSERT(ggml_is_padded_1d(a));
GGML_ASSERT(ggml_is_contiguous_rows(a));
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
src/llama-arch.cpp
+1
View File
@@ -1347,6 +1347,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MELLUM: // [TAG_MOE_AMAX_SCALE]
case LLM_ARCH_FLINT: // [TAG_MOE_AMAX_SCALE]
case LLM_ARCH_LAGUNA: // [TAG_MOE_AMAX_SCALE]
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_KIMI_K3:
src/llama-context.cpp
+9 -8
View File
@@ -489,6 +489,7 @@ llama_context::llama_context(
/*.swa_full =*/ params.swa_full,
/*.ctx_type =*/ cparams.ctx_type,
/*.mem_other =*/ llama_get_memory(cparams.ctx_other),
/*.no_alloc =*/ model.no_alloc(),
};
memory.reset(model.create_memory(params_mem, cparams));
@@ -559,7 +560,7 @@ llama_context::llama_context(
}
if (cparams.moe_cache_size > 0) {
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
}
sched_reserve();
@@ -587,7 +588,7 @@ llama_context::~llama_context() {
synchronize();
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
if (!model.hparams.no_alloc && !opt_ctx) {
if (!model.no_alloc() && !opt_ctx) {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
@@ -788,7 +789,7 @@ void llama_context::sched_reserve() {
// reserve pp (prompt processing) graph first so that buffers are only allocated once
{
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr);
if (!gf) {
if (cparams.pipeline_parallel) {
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
@@ -815,7 +816,7 @@ void llama_context::sched_reserve() {
n_splits_tg = n_splits_pp;
n_nodes_tg = n_nodes_pp;
} else {
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc());
if (!gf) {
throw std::runtime_error("failed to allocate compute tg buffers");
}
@@ -837,10 +838,10 @@ void llama_context::sched_reserve() {
// [TAG_RESERVE_DIAG_DECAY]
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.no_alloc());
break;
default:
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc());
};
if (!gf) {
@@ -851,7 +852,7 @@ void llama_context::sched_reserve() {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
if (!model.hparams.no_alloc) {
if (!model.no_alloc()) {
backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
}
if (backend_buf_exp_size[i] > 1) {
@@ -3730,7 +3731,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
ret[buft].context += size;
}
}
if (model.hparams.no_alloc) {
if (model.no_alloc()) {
for (size_t i = 0; i < backends.size(); ++i) {
ggml_backend_t backend = backends[i].get();
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
src/llama-graph.cpp
+1 -1
View File
@@ -2519,7 +2519,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
// ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922
llm_graph_amax_src1 amax_src1 = { nullptr, nullptr };
if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) {
if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_LAGUNA) {
amax_src1 = build_amax_src1_f16(cur);
cur = amax_src1.res;
}
src/llama-hparams.h
-1
View File
@@ -61,7 +61,6 @@ struct llama_hparams {
// for example: n_embd_out() vs n_embd_out_impl
bool vocab_only;
bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams
bool rope_finetuned;
bool use_par_res;
bool swin_norm;
src/llama-kv-cache-dsa-iswa.cpp
+5 -5
View File
@@ -13,6 +13,7 @@
llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -22,7 +23,6 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
const layer_filter_cb & filter_mla,
const layer_filter_cb & filter_lid,
const layer_reuse_cb & reuse) : unified(unified) {
@@ -63,15 +63,15 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa);
kv_dsa = std::make_unique<llama_kv_cache_dsa>(
model, type_k, type_v,
v_trans, offload, unified, size_dsa, n_seq_max, n_pad,
model, no_alloc, type_k, type_v,
v_trans, offload, unified, size_dsa, n_seq_max,
0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse);
LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
kv_swa = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max, n_pad,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max,
hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr);
}
src/llama-kv-cache-dsa-iswa.h
+1 -1
View File
@@ -14,6 +14,7 @@ class llama_kv_cache_dsa_iswa : public llama_memory_i {
public:
llama_kv_cache_dsa_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -23,7 +24,6 @@ public:
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
const layer_filter_cb & filter_mla,
const layer_filter_cb & filter_lid,
const layer_reuse_cb & reuse);
src/llama-kv-cache-dsa.cpp
+5 -5
View File
@@ -13,6 +13,7 @@
llama_kv_cache_dsa::llama_kv_cache_dsa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -20,7 +21,6 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
const layer_filter_cb & filter_mla,
@@ -31,8 +31,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
kv_mla = std::make_unique<llama_kv_cache>(
model, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
model, no_alloc, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max,
n_swa, swa_type, nullptr, filter_mla, reuse, nullptr);
// we use llama_kv_cache for caching indexer keys
@@ -48,8 +48,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
kv_lid = std::make_unique<llama_kv_cache>(
model, hparams_lid, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
model, no_alloc, hparams_lid, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max,
n_swa, swa_type, nullptr, filter_lid, reuse, nullptr);
}
src/llama-kv-cache-dsa.h
+1 -1
View File
@@ -16,6 +16,7 @@ class llama_kv_cache_dsa : public llama_memory_i {
public:
llama_kv_cache_dsa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -23,7 +24,6 @@ public:
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
const layer_filter_cb & filter_mla,
src/llama-kv-cache-dsv4.cpp
+27 -50
View File
@@ -931,21 +931,24 @@ static void dsv4_make_k_only(llama_hparams & hparams) {
llama_dsv4_comp_state::llama_dsv4_comp_state(
const llama_model & model,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
bool no_alloc,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
const llama_memory_i::layer_filter_cb & filter) :
hparams(model.hparams),
ratio(ratio),
state_size(state_size),
n_embd_state(n_embd_state),
n_stream(unified ? 1 : n_seq_max),
n_rs_seq(n_rs_seq) {
n_rs_seq(n_rs_seq),
ctxs_bufs(no_alloc) {
const llama_hparams & hparams = model.hparams;
struct ggml_backend_buft_comparator {
bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {
return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;
@@ -1020,25 +1023,10 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
}
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for DSV4 compressor state");
}
ggml_backend_buffer_clear(buf, 0);
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "DSV4 compressor state");
LLAMA_LOG_INFO("%s: %10s DSV4 %s state buffer size = %8.2f MiB\n",
__func__, ggml_backend_buffer_name(buf), name, ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n",
@@ -1119,18 +1107,7 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
void llama_dsv4_comp_state::state_write(
@@ -1265,6 +1242,7 @@ size_t llama_dsv4_comp_state::total_size() const {
llama_kv_cache_dsv4::llama_kv_cache_dsv4(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -1274,7 +1252,6 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
uint32_t n_rs_seq,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse) :
@@ -1309,8 +1286,8 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
dsv4_make_k_only(hparams_raw);
kv_raw = std::make_unique<llama_kv_cache_iswa>(
model, hparams_raw, type_k, type_v,
v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad,
model, no_alloc, hparams_raw, type_k, type_v,
v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch,
nullptr, filter_raw, reuse, nullptr);
// V4.1: CSA slot = ratio 2, HCA slot = ratio 1, no overlap and no indexer compressor; the index K is stored
@@ -1413,24 +1390,24 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
__func__, dsv4_comp_size(kv_size, ratio_csa));
kv_csa = std::make_unique<llama_kv_cache>(
model, hparams_csa, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
model, no_alloc, hparams_csa, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n",
__func__, dsv4_comp_size(kv_size, ratio_hca));
kv_hca = std::make_unique<llama_kv_cache>(
model, hparams_hca, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad,
model, no_alloc, hparams_hca, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n",
__func__, dsv4_comp_size(kv_size, ratio_csa));
kv_lid = std::make_unique<llama_kv_cache>(
model, hparams_lid, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
model, no_alloc, hparams_lid, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__);
@@ -1439,19 +1416,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k();
csa_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
model, no_alloc, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
csa_state_embd, n_rs_seq, "csa", filter_csa_state);
LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__);
hca_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
model, no_alloc, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca_state);
LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__);
lid_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
model, no_alloc, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid);
// DSV4 attention reads compressed-K / compressor-state rows that the current
src/llama-kv-cache-dsv4.h
+11 -12
View File
@@ -16,14 +16,15 @@ public:
llama_dsv4_comp_state(
const llama_model & model,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
bool no_alloc,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
const llama_memory_i::layer_filter_cb & filter);
void clear(llama_seq_id seq_id, bool data);
@@ -61,15 +62,13 @@ private:
std::vector<ggml_tensor *> score_stream;
};
const llama_hparams & hparams;
const uint32_t ratio;
const uint32_t state_size;
const uint32_t n_embd_state;
const uint32_t n_stream;
const uint32_t n_rs_seq;
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
llama_memory_bufs ctxs_bufs;
std::vector<layer> layers;
@@ -92,6 +91,7 @@ class llama_kv_cache_dsv4 : public llama_memory_i {
public:
llama_kv_cache_dsv4(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -101,7 +101,6 @@ public:
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
uint32_t n_rs_seq,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse);
src/llama-kv-cache-iswa.cpp
+9 -9
View File
@@ -13,6 +13,7 @@
llama_kv_cache_iswa::llama_kv_cache_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -22,18 +23,18 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
llama_memory_t mem_other,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
const layer_share_cb & share) :
llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) {
llama_kv_cache_iswa(model, no_alloc, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
kv_size, n_seq_max, n_ubatch, mem_other, filter, reuse, share) {
}
llama_kv_cache_iswa::llama_kv_cache_iswa(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -43,7 +44,6 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
llama_memory_t mem_other,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
@@ -93,15 +93,15 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
}
kv_base = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
v_trans, offload, unified, size_base, n_seq_max, n_pad,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_base, n_seq_max,
0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share);
LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
kv_swa = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max, n_pad,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max,
hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share);
}
src/llama-kv-cache-iswa.h
+3 -3
View File
@@ -15,6 +15,7 @@ class llama_kv_cache_iswa : public llama_memory_i {
public:
llama_kv_cache_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -24,7 +25,6 @@ public:
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
llama_memory_t mem_other,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
@@ -32,7 +32,8 @@ public:
llama_kv_cache_iswa(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -42,7 +43,6 @@ public:
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_ubatch,
uint32_t n_pad,
llama_memory_t mem_other,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
src/llama-kv-cache-msa.cpp
+8 -8
View File
@@ -12,6 +12,7 @@
llama_kv_cache_msa::llama_kv_cache_msa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -19,21 +20,20 @@ llama_kv_cache_msa::llama_kv_cache_msa(
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
const layer_filter_cb & filter,
const layer_filter_cb & filter_idx,
const layer_reuse_cb & reuse) :
hparams_idx(model.hparams),
n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_pad(n_pad),
n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max),
n_swa(n_swa), swa_type(swa_type) {
LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
kv_base = std::make_unique<llama_kv_cache>(
model, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
model, no_alloc, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max,
n_swa, swa_type, nullptr, filter, reuse, nullptr);
// the MSA indexer uses a single key head per layer
@@ -44,8 +44,8 @@ llama_kv_cache_msa::llama_kv_cache_msa(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
kv_idx = std::make_unique<llama_kv_cache>(
model, hparams_idx, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
model, no_alloc, hparams_idx, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max,
n_swa, swa_type, nullptr, filter_idx, reuse, nullptr);
}
@@ -261,7 +261,7 @@ const llama_kv_cache_context * llama_kv_cache_msa_context::get_idx() const {
uint32_t llama_kv_cache_msa_context::get_n_pos() const {
// pad the value so that the graph remains constant across batches and can be reused
const uint32_t n_pad_cur = std::max(kv->get_n_pad(), 256u);
const uint32_t n_pad = 256;
llama_pos pos_max = -1;
@@ -269,7 +269,7 @@ uint32_t llama_kv_cache_msa_context::get_n_pos() const {
pos_max = std::max(pos_max, kv->seq_pos_max(seq_id));
}
return std::max(n_pad_cur, GGML_PAD((uint32_t) (pos_max + 1), n_pad_cur));
return std::max(n_pad, GGML_PAD((uint32_t) (pos_max + 1), n_pad));
}
void llama_kv_cache_msa_context::set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const {
src/llama-kv-cache-msa.h
+1 -3
View File
@@ -15,6 +15,7 @@ class llama_kv_cache_msa : public llama_memory_i {
public:
llama_kv_cache_msa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -22,7 +23,6 @@ public:
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
const layer_filter_cb & filter,
@@ -67,7 +67,6 @@ public:
llama_kv_cache * get_base() const;
llama_kv_cache * get_idx () const;
uint32_t get_n_pad() const { return n_pad; }
uint32_t get_n_seq_max() const { return n_seq_max; }
uint32_t get_n_swa() const { return n_swa; }
llama_swa_type get_swa_type() const { return swa_type; }
@@ -78,7 +77,6 @@ private:
const uint32_t n_stream = 1;
const uint32_t n_seq_max = 1;
const uint32_t n_pad = 1;
const uint32_t n_swa = 0;
const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
src/llama-kv-cache.cpp
+10 -38
View File
@@ -64,7 +64,8 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) {
llama_kv_cache::llama_kv_cache(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -72,16 +73,16 @@ llama_kv_cache::llama_kv_cache(
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
llama_memory_t mem_other,
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
const layer_share_cb & share,
const char * name_tag) :
const char * name_tag) :
model(model), hparams(hparams), v_trans(v_trans),
n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type),
n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), swa_type(swa_type),
ctxs_bufs(no_alloc),
other(static_cast<llama_kv_cache *>(mem_other)),
v_cells_impl(other ? other->v_cells_impl : std::make_shared<llama_kv_cells_vec>()),
v_cells(*v_cells_impl) {
@@ -97,8 +98,6 @@ llama_kv_cache::llama_kv_cache(
}
}
GGML_ASSERT(kv_size % n_pad == 0);
const uint32_t n_layer = hparams.n_layer_all;
// define a comparator for the buft -> ctx map to ensure that the order is well-defined:
@@ -273,25 +272,11 @@ llama_kv_cache::llama_kv_cache(
}
}
// allocate tensors and initialize the buffers to avoid NaNs in the padding
// allocate the tensors and initialize the buffers to avoid NaNs in the padding
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for kv cache");
}
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "kv cache");
LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ggml_backend_buffer_clear(buf, 0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
{
@@ -705,20 +690,7 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
// GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
llama_memory_context_ptr llama_kv_cache::init_batch(
@@ -1285,12 +1257,12 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
// pad the n_kv value so that the graph remains constant across batches and can be reused
// note: this also helps some backends with performance (f.ex https://github.com/ggml-org/llama.cpp/pull/16812#issuecomment-3455112220)
const uint32_t n_pad_cur = std::max(n_pad, 256u);
const uint32_t n_pad = 256;
for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
const auto & cells = v_cells[sinfo.strm[s]];
result = std::max(std::min(cells.size(), std::max(n_pad_cur, GGML_PAD(cells.used_max_p1(), n_pad_cur))), result);
result = std::max(std::min(cells.size(), std::max(n_pad, GGML_PAD(cells.used_max_p1(), n_pad))), result);
}
return result;
src/llama-kv-cache.h
+2 -5
View File
@@ -98,6 +98,7 @@ public:
// likely through `struct llama_memory_params`
llama_kv_cache(
const llama_model & model,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
@@ -106,7 +107,6 @@ public:
bool unified,
uint32_t kv_size,
uint32_t n_seq_max,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
llama_memory_t mem_other,
@@ -271,9 +271,6 @@ private:
const uint32_t n_seq_max = 1;
const uint32_t n_stream = 1;
// required padding
const uint32_t n_pad = 1;
// SWA
const uint32_t n_swa = 0;
@@ -296,7 +293,7 @@ private:
const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
// ggml contexts for the KV cache along with the allocated backend buffers:
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
llama_memory_bufs ctxs_bufs;
// the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
// note: this is not part of the KV state and it's only used to speed-up the find_slot() method
src/llama-memory-hybrid-idx.cpp
+6 -9
View File
@@ -1,20 +1,16 @@
#include "llama-memory-hybrid-idx.h"
#include <algorithm>
#include <cmath>
#include <type_traits>
#include "llama-impl.h"
#include "llama-batch.h"
#include "llama-io.h"
#include "llama-model.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <iterator>
#include <stdexcept>
#include <type_traits>
//
// llama_memory_hybrid_idx
@@ -22,12 +18,12 @@
llama_memory_hybrid_idx::llama_memory_hybrid_idx(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
bool v_trans,
uint32_t kv_size,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
/* recurrent */
@@ -45,7 +41,8 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
const layer_filter_cb & filter_idx) :
llama_memory_hybrid(
model,
type_k, type_v, v_trans, kv_size, n_pad, n_swa, swa_type,
no_alloc,
type_k, type_v, v_trans, kv_size, n_swa, swa_type,
type_r, type_s, rs_size,
n_seq_max, n_rs_seq, offload, unified,
filter_attn, filter_recr),
@@ -68,8 +65,8 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
return new llama_kv_cache(
model, hparams_idx, type_k, type_v, v_trans, offload, unified,
kv_size, n_seq_max, n_pad, n_swa, swa_type,
model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified,
kv_size, n_seq_max, n_swa, swa_type,
nullptr, filter_idx, nullptr, nullptr, "idx_");
}()) {}
src/llama-memory-hybrid-idx.h
+1 -1
View File
@@ -18,12 +18,12 @@ class llama_memory_hybrid_idx : public llama_memory_hybrid {
public:
llama_memory_hybrid_idx(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
bool v_trans,
uint32_t kv_size,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
/* recurrent */
src/llama-memory-hybrid-iswa.cpp
+3 -2
View File
@@ -10,6 +10,7 @@
llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
@@ -17,7 +18,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
bool swa_full,
uint32_t kv_size,
uint32_t n_ubatch,
uint32_t n_pad,
/* recurrent */
ggml_type type_r,
ggml_type type_s,
@@ -33,6 +33,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
hparams(model.hparams),
mem_attn(new llama_kv_cache_iswa(
model,
no_alloc,
type_k,
type_v,
v_trans,
@@ -42,7 +43,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
kv_size,
n_seq_max,
n_ubatch,
n_pad,
nullptr,
filter_attn == nullptr ?
[&](int32_t il) { return !hparams.is_recr(il); }
@@ -52,6 +52,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
)),
mem_recr(new llama_memory_recurrent(
model,
no_alloc,
type_r,
type_s,
offload,
src/llama-memory-hybrid-iswa.h
+1 -1
View File
@@ -20,6 +20,7 @@ class llama_memory_hybrid_iswa : public llama_memory_i {
public:
llama_memory_hybrid_iswa(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
@@ -27,7 +28,6 @@ public:
bool swa_full,
uint32_t kv_size,
uint32_t n_ubatch,
uint32_t n_pad,
/* recurrent */
ggml_type type_r,
ggml_type type_s,
src/llama-memory-hybrid.cpp
+3 -2
View File
@@ -10,12 +10,12 @@
llama_memory_hybrid::llama_memory_hybrid(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
bool v_trans,
uint32_t kv_size,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
/* recurrent */
@@ -33,6 +33,7 @@ llama_memory_hybrid::llama_memory_hybrid(
hparams(model.hparams),
mem_attn(new llama_kv_cache(
model,
no_alloc,
model.hparams,
type_k,
type_v,
@@ -41,7 +42,6 @@ llama_memory_hybrid::llama_memory_hybrid(
unified,
kv_size,
n_seq_max,
n_pad,
n_swa,
swa_type,
nullptr,
@@ -53,6 +53,7 @@ llama_memory_hybrid::llama_memory_hybrid(
)),
mem_recr(new llama_memory_recurrent(
model,
no_alloc,
type_r,
type_s,
offload,
src/llama-memory-hybrid.h
+1 -1
View File
@@ -20,12 +20,12 @@ class llama_memory_hybrid : public llama_memory_i {
public:
llama_memory_hybrid(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
bool v_trans,
uint32_t kv_size,
uint32_t n_pad,
uint32_t n_swa,
llama_swa_type swa_type,
/* recurrent */
src/llama-memory-recurrent.cpp
+6 -28
View File
@@ -19,13 +19,14 @@
llama_memory_recurrent::llama_memory_recurrent(
const llama_model & model,
bool no_alloc,
ggml_type type_r,
ggml_type type_s,
bool offload,
uint32_t mem_size,
uint32_t n_seq_max,
uint32_t n_rs_seq,
const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) {
const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max), ctxs_bufs(no_alloc) {
const int32_t n_layer = hparams.n_layer();
head = 0;
@@ -114,23 +115,11 @@ llama_memory_recurrent::llama_memory_recurrent(
}
}
// allocate tensors and initialize the buffers to avoid NaNs in the padding
// allocate the tensors and initialize the buffers to avoid NaNs in the padding
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for rs cache");
}
ggml_backend_buffer_clear(buf, 0);
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "rs cache");
LLAMA_LOG_INFO("%s: %10s RS buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
if (is_empty()) {
@@ -442,18 +431,7 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
src/llama-memory-recurrent.h
+3 -2
View File
@@ -18,6 +18,7 @@ class llama_memory_recurrent : public llama_memory_i {
public:
llama_memory_recurrent(
const llama_model & model,
bool no_alloc,
ggml_type type_r,
ggml_type type_s,
bool offload,
@@ -120,8 +121,8 @@ private:
const uint32_t n_seq_max = 1;
// ggml contexts for the KV cache along with the allocated backend buffers:
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
// ggml contexts for the RS cache along with the allocated backend buffers:
llama_memory_bufs ctxs_bufs;
// true if no layers - can happen if the layer filter removes all layers
bool is_empty() const;
src/llama-memory.cpp
+52
View File
@@ -4,6 +4,8 @@
#include "llama-io.h"
#include <set>
#include <stdexcept>
#include <string>
#include <vector>
llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) {
@@ -64,6 +66,52 @@ bool llama_memory_status_is_fail(llama_memory_status status) {
return false;
}
//
// llama_memory_bufs
//
ggml_backend_buffer_ptr llama_memory_bufs::alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name) {
ggml_backend_buffer_ptr buf;
if (no_alloc) {
buf.reset(ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0)); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
t->buffer = buf.get(); // set dummy buffer so that the backend scheduler won't try to allocate the tensors
}
} else {
buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft)); // real buffer
}
if (!buf) {
throw std::runtime_error(std::string("failed to allocate buffer for ") + name);
}
return buf;
}
ggml_backend_buffer_t llama_memory_bufs::emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name) {
ggml_backend_buffer_ptr buf = alloc_buf(ctx.get(), buft, no_alloc, name);
// initialize the buffer to avoid NaNs in the padding
ggml_backend_buffer_clear(buf.get(), 0);
bufs.emplace_back(std::move(ctx), std::move(buf));
return bufs.back().second.get();
}
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
}
// collects the buffers of the tensors that a state write visits, without keeping the data
struct llama_memory_buffers_writer : public llama_io_write_i {
std::set<ggml_backend_buffer_t> buffers;
@@ -84,6 +132,10 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
};
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
if (mem == nullptr) {
return {};
}
llama_memory_buffers_writer writer;
mem->state_write(writer);
src/llama-memory.h
+31
View File
@@ -6,6 +6,7 @@
#include <map>
#include <memory>
#include <functional>
#include <vector>
struct llama_ubatch;
@@ -25,6 +26,36 @@ struct llama_memory_params {
llama_context_type ctx_type;
llama_memory_t mem_other;
// only simulate the memory allocations
bool no_alloc;
};
// the tensor contexts and backend buffers of a memory instance
// with no_alloc, dummy buffers are attached instead of allocating the tensor data, so the memory use can still be measured
struct llama_memory_bufs {
explicit llama_memory_bufs(bool no_alloc) : no_alloc(no_alloc) {}
// allocate the tensors of ctx on buft, or attach a dummy buffer if no_alloc is set
// returns the buffer, for logging
ggml_backend_buffer_t emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name);
// the memory used by the tensors: the buffer sizes, or the simulated sizes if no_alloc is set
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
// allocate a single ctx buffer - for the memory instances that keep their own buffer layout
static ggml_backend_buffer_ptr alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name);
bool empty() const { return bufs.empty(); }
auto begin() { return bufs.begin(); }
auto end() { return bufs.end(); }
auto begin() const { return bufs.begin(); }
auto end() const { return bufs.end(); }
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> bufs;
const bool no_alloc;
};
enum llama_memory_status {
src/llama-model.cpp
+24 -19
View File
@@ -2448,10 +2448,14 @@ llama_split_mode llama_model::split_mode() const {
return params.split_mode;
}
bool llama_model::no_alloc() const {
return params.no_alloc;
}
std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, bufs] : pimpl->ctxs_bufs) {
if (hparams.no_alloc) {
if (no_alloc()) {
GGML_ASSERT(bufs.size() == 1);
ggml_backend_buffer_t buf = bufs[0].get();
GGML_ASSERT(ggml_backend_buffer_get_base(buf) == nullptr);
@@ -2506,7 +2510,7 @@ void llama_model::print_info() const {
// hparams
LLAMA_LOG_INFO("%s: arch = %s\n", __func__, arch_name().c_str());
LLAMA_LOG_INFO("%s: vocab_only = %d\n", __func__, hparams.vocab_only);
LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, hparams.no_alloc);
LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, no_alloc());
if (!hparams.vocab_only) {
LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train);
@@ -2830,6 +2834,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_msa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2837,7 +2842,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
@@ -2856,6 +2860,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -2864,7 +2869,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
@@ -2882,6 +2886,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2889,7 +2894,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
filter_mla,
@@ -2923,11 +2927,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_memory_hybrid_idx(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
/* attn_kv_size */ cparams.n_ctx_seq,
/* attn_n_pad */ 1,
/* attn_n_swa */ hparams.n_swa,
/* attn_swa_type */ hparams.swa_type,
/* recurrent_type_r */ GGML_TYPE_F32,
@@ -2947,6 +2951,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// full-attention checkpoint: no indexer, so no indexer key cache
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -2955,7 +2960,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
@@ -2968,6 +2972,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2975,7 +2980,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
@@ -2994,6 +2998,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -3002,7 +3007,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
@@ -3019,6 +3023,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3028,7 +3033,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
filter_mla,
filter_lid,
nullptr);
@@ -3046,6 +3050,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3055,7 +3060,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
nullptr,
filter_mtp,
nullptr,
@@ -3063,6 +3067,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_kv_cache_dsv4(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3072,7 +3077,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
cparams.n_rs_seq,
nullptr,
nullptr);
@@ -3086,6 +3090,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3095,7 +3100,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
nullptr,
nullptr,
nullptr,
@@ -3120,6 +3124,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
if (llm_arch_is_recurrent(arch)) {
res = new llama_memory_recurrent(
*this,
params.no_alloc,
GGML_TYPE_F32,
GGML_TYPE_F32,
cparams.offload_kqv,
@@ -3175,13 +3180,13 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// Use hybrid-iswa for hybrid models with SWA
res = new llama_memory_hybrid_iswa(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
/* attn_swa_full */ params.swa_full,
/* attn_kv_size */ cparams.n_ctx_seq,
/* attn_n_ubatch */ cparams.n_ubatch,
/* attn_n_pad */ 1,
/* recurrent_type_r */ GGML_TYPE_F32,
/* recurrent_type_s */ GGML_TYPE_F32,
/* recurrent_rs_size */ std::max((uint32_t) 1, cparams.n_seq_max),
@@ -3195,11 +3200,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// sparse attention over a per-token indexer cache, in its own memory type
res = new llama_memory_hybrid_idx(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
/* attn_kv_size */ cparams.n_ctx_seq,
/* attn_n_pad */ 1,
/* attn_n_swa */ hparams.n_swa,
/* attn_swa_type */ hparams.swa_type,
/* recurrent_type_k */ GGML_TYPE_F32,
@@ -3215,11 +3220,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_memory_hybrid(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
/* attn_kv_size */ cparams.n_ctx_seq,
/* attn_n_pad */ 1,
/* attn_n_swa */ hparams.n_swa,
/* attn_swa_type */ hparams.swa_type,
/* recurrent_type_k */ GGML_TYPE_F32,
@@ -3281,6 +3286,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3290,7 +3296,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
mem_other,
filter,
reuse,
@@ -3298,6 +3303,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3307,7 +3313,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
nullptr,
filter,
reuse,
@@ -3318,6 +3323,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -3326,7 +3332,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
1,
hparams.n_swa,
hparams.swa_type,
nullptr,
src/llama-model.h
+1
View File
@@ -886,6 +886,7 @@ struct llama_model {
uint32_t n_gpu_layers() const;
llama_split_mode split_mode() const;
bool no_alloc() const;
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
src/llama-moe-cache.cpp
+13 -30
View File
@@ -1,6 +1,7 @@
#include "llama-moe-cache.h"
#include "llama-impl.h"
#include "llama-memory.h"
#include "llama-model.h"
#include "ggml-cpp.h"
@@ -252,7 +253,7 @@ struct llama_moe_cache::impl {
// views used by copy_experts
ggml_context_ptr ctx_views;
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
impl(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) {
for (size_t i = 0; i < backends.size(); ++i) {
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
@@ -418,37 +419,19 @@ struct llama_moe_cache::impl {
devices[g.id].buf_size += alloc_size(g, g.n_slots);
}
if (model.hparams.no_alloc) {
// only used to measure the memory use, see llama_context::memory_breakdown
for (device & d : devices) {
if (!d.ctx) {
continue;
}
d.buf.reset(ggml_backend_buft_alloc_buffer(d.buft, 0));
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
t->buffer = d.buf.get();
}
// with no_alloc, dummy buffers are attached so the memory use can still be measured, see llama_context::memory_breakdown
for (device & d : devices) {
if (!d.ctx) {
continue;
}
buf_host.reset(ggml_backend_buft_alloc_buffer(buft_host, 0));
for (ggml_tensor * t = ggml_get_first_tensor(ctx_host.get()); t != nullptr; t = ggml_get_next_tensor(ctx_host.get(), t)) {
t->buffer = buf_host.get();
}
} else {
for (device & d : devices) {
if (!d.ctx) {
continue;
}
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
if (!d.buf) {
throw std::runtime_error("failed to allocate the MoE cache buffers");
}
d.buf = llama_memory_bufs::alloc_buf(d.ctx.get(), d.buft, no_alloc, "MoE cache");
if (!no_alloc) {
ggml_backend_buffer_clear(d.buf.get(), 0);
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
}
buf_host.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_host.get(), buft_host));
if (!buf_host) {
throw std::runtime_error("failed to allocate the MoE cache buffers");
}
}
buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps");
if (!no_alloc) {
ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
buf_host_size = ggml_backend_buffer_get_size(buf_host.get());
@@ -633,8 +616,8 @@ struct llama_moe_cache::impl {
}
};
llama_moe_cache::llama_moe_cache(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
pimpl(new impl(model, backends, bufts, size)) {
llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
pimpl(new impl(model, no_alloc, backends, bufts, size)) {
}
llama_moe_cache::~llama_moe_cache() = default;
src/llama-moe-cache.h
+1 -1
View File
@@ -13,7 +13,7 @@ struct llama_model;
class llama_moe_cache {
public:
// backends are all the backends of the context, each GPU gets its own cache of the given size for the layers assigned to it
llama_moe_cache(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size);
llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size);
~llama_moe_cache();
// the device that caches layer il
src/llama.cpp
-1
View File
@@ -342,7 +342,6 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * meta
model->t_start_us = tm.t_start_us;
model->hparams.vocab_only = params.vocab_only;
model->hparams.no_alloc = params.no_alloc;
try {
model->load_hparams(ml);
src/models/deepseek4.cpp
+38 -40
View File
@@ -342,24 +342,23 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn(
return comb;
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
llama_model_deepseek4::graph::hc_pre
llama_model_deepseek4::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const {
return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il);
hc_mixes mixes = build_hc_mixes(x, hc_fn, hc_scale, hc_base, il);
return { build_hc_pre(x, mixes.pre, il), mixes.post, mixes.comb };
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
llama_model_deepseek4::graph::hc_mixes
llama_model_deepseek4::graph::build_hc_mixes(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
@@ -386,28 +385,29 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
cb(pre, "hc_pre", il);
*post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
*post = dsv4_hc_affine(ctx0, *post, scale_post, base_post);
*post = ggml_sigmoid(ctx0, *post);
*post = ggml_scale(ctx0, *post, 2.0f);
cb(*post, "hc_post", il);
ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
post = dsv4_hc_affine(ctx0, post, scale_post, base_post);
post = ggml_sigmoid(ctx0, post);
post = ggml_scale(ctx0, post, 2.0f);
cb(post, "hc_post", il);
ggml_tensor * comb = nullptr;
if (cparams.fused_dsv4_hc_comb) {
*comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
(int32_t) hparams.dsv4_hc_sinkhorn_iters);
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il});
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il});
} else {
ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2);
ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc);
*comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
*comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb);
*comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
*comb = build_hc_sinkhorn(*comb, il);
comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb);
comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt);
comb = build_hc_sinkhorn(comb, il);
}
cb(*comb, "hc_comb", il);
cb(comb, "hc_comb", il);
return pre;
return { pre, post, comb };
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_post(
@@ -1252,14 +1252,12 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
}
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
cur = build_hc_pre(inpL,
const hc_pre attn_hc = build_hc_pre(inpL,
model.layers[il].hc_attn_fn,
model.layers[il].hc_attn_scale,
model.layers[il].hc_attn_base,
&post, &comb, il);
il);
cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1267,20 +1265,21 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
cur = build_attention(model, inp_dsv4, cur, inp_pos, il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
cur = build_hc_pre(inpL,
const hc_pre ffn_hc = build_hc_pre(inpL,
model.layers[il].hc_ffn_fn,
model.layers[il].hc_ffn_scale,
model.layers[il].hc_ffn_base,
&post, &comb, il);
il);
cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
ggml_build_forward_expand(gf, residual);
ggml_build_forward_expand(gf, post);
ggml_build_forward_expand(gf, comb);
ggml_build_forward_expand(gf, ffn_hc.post);
ggml_build_forward_expand(gf, ffn_hc.comb);
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -1330,7 +1329,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_last", il);
}
@@ -1430,14 +1429,12 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cb(inpL, "mtp_eh_proj", il);
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
ggml_tensor * cur = build_hc_pre(inpL,
hc_pre attn_hc = build_hc_pre(inpL,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
&post, &comb, il);
il);
ggml_tensor * cur = attn_hc.cur;
cb(cur, "mtp_hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1445,15 +1442,16 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = build_attention(model, inp_attn, cur, inp_pos, il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "mtp_hc_attn_post", il);
residual = inpL;
cur = build_hc_pre(inpL,
hc_pre ffn_hc = build_hc_pre(inpL,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
&post, &comb, il);
il);
cur = ffn_hc.cur;
cb(cur, "mtp_hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1483,7 +1481,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "mtp_ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "mtp_l_out", il);
src/models/deepseek41.cpp
+8 -10
View File
@@ -847,10 +847,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
}
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
// the first layer's attention collapses with the identity pre-mix: copy 0
cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
@@ -861,18 +858,19 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
pre_prev = ffn_mix.pre;
cur = build_hc_pre(inpL, attn_pre, il);
cur = build_hc_pre(inpL, attn_mix.pre, il);
cb(cur, "hc_ffn_pre", il);
ggml_build_forward_expand(gf, residual);
ggml_build_forward_expand(gf, post);
ggml_build_forward_expand(gf, comb);
ggml_build_forward_expand(gf, ffn_mix.post);
ggml_build_forward_expand(gf, ffn_mix.comb);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -912,7 +910,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_out", il);
}
src/models/dflash.cpp
+14 -19
View File
@@ -1204,14 +1204,12 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
ggml_tensor * cur = build_hc_pre(inpL,
hc_pre attn_hc = build_hc_pre(inpL,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
&post, &comb, il);
il);
ggml_tensor * cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1219,15 +1217,16 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
cur = build_attention(model, inp_attn, cur, inp_pos, il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
cur = build_hc_pre(inpL,
hc_pre ffn_hc = build_hc_pre(inpL,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
&post, &comb, il);
il);
cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1256,7 +1255,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
cb(inpL, "l_out", il);
}
@@ -1403,10 +1402,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
ggml_tensor * cur;
if (pre_prev == nullptr) {
@@ -1461,13 +1457,14 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
cb(cur, "attn_out", il);
}
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
pre_prev = ffn_mix.pre;
cur = build_hc_pre(inpL, attn_pre, il);
cur = build_hc_pre(inpL, attn_mix.pre, il);
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1496,13 +1493,11 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il);
ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
cb(l_out, "l_out", il);
ggml_build_forward_expand(gf, l_out);
pre_prev = ffn_pre;
}
// no hc_head: collapse with the last FFN's pre-mix
src/models/glm5-next.cpp
+34 -34
View File
@@ -442,13 +442,12 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_sinkhorn(
return comb;
}
ggml_tensor * llama_model_glm5_next::graph::build_hc_pre(
llama_model_glm5_next::graph::hc_pre
llama_model_glm5_next::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
@@ -475,29 +474,29 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_pre(
pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
cb(pre, "hc_pre", il);
*post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
*post = dsv4_hc_affine(ctx0, *post, scale_post, base_post);
*post = ggml_sigmoid(ctx0, *post);
*post = ggml_scale(ctx0, *post, 2.0f);
cb(*post, "hc_post", il);
ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
post = dsv4_hc_affine(ctx0, post, scale_post, base_post);
post = ggml_sigmoid(ctx0, post);
post = ggml_scale(ctx0, post, 2.0f);
cb(post, "hc_post", il);
ggml_tensor * comb = nullptr;
if (cparams.fused_dsv4_hc_comb) {
*comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
(int32_t) hparams.dsv4_hc_sinkhorn_iters);
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il});
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il});
} else {
ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2);
ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc);
*comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
*comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb);
*comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
*comb = build_hc_sinkhorn(*comb, il);
comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb);
comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt);
comb = build_hc_sinkhorn(comb, il);
}
cb(*comb, "hc_comb", il);
cb(comb, "hc_comb", il);
ggml_tensor * result = build_hc_pre(x, pre, il);
return result;
return { build_hc_pre(x, pre, il), post, comb };
}
ggml_tensor * llama_model_glm5_next::graph::build_hc_post(
@@ -609,8 +608,9 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "mtp_attn_norm", il);
ggml_tensor * prev_sel = nullptr;
cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);
// the NextN block has a full indexer, nothing to carry in
dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, nullptr, il);
cur = dsa.cur;
cb(cur, "mtp_attn_out", il);
// narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows
@@ -711,10 +711,8 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
cur = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -725,14 +723,17 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda,
d_inner, n_seq_tokens, n_seqs, il);
} else {
cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);
dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, prev_sel, il);
cur = dsa.cur;
prev_sel = dsa.sel;
}
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
cur = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -770,7 +771,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
cb(cur, "ffn_out", il);
}
inpL = build_hc_post(cur, residual, post, comb, il);
inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_out", il);
}
@@ -1039,10 +1040,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
// Nope MLA layer with sparse attention over the indexer selection
ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
llama_model_glm5_next::graph::dsa_out
llama_model_glm5_next::graph::build_dsa_layer(
ggml_tensor * cur, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il) {
llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il) {
const auto * mctx_mla = mctx_hyb->get_attn();
@@ -1073,13 +1075,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
ggml_tensor * kq_mask = inp_attn->get_kq_mask();
ggml_tensor * sel = nullptr;
ggml_tensor * sel = prev_sel;
if (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il)) { // the NextN block always has a full indexer
sel = build_kpool_select(cur, qr, kq_mask, layer, mctx_hyb, inp_kpool, il);
*prev_sel = sel;
} else {
GGML_ASSERT(*prev_sel != nullptr && "shared indexer layer must follow a full indexer layer");
sel = *prev_sel;
GGML_ASSERT(sel != nullptr && "shared indexer layer must follow a full indexer layer");
}
ggml_build_forward_expand(gf, q_absorbed);
@@ -1099,5 +1099,5 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
out = ggml_mul_mat(ctx0, layer.wo, out);
cb(out, "attn_out", il);
return out;
return { out, sel };
}
src/models/hy-v4.cpp
+34 -29
View File
@@ -172,12 +172,12 @@ static ggml_tensor * hy_v4_hc_reduce(ggml_context * ctx0, ggml_tensor * x, ggml_
return ggml_cast(ctx0, result, out_type);
}
ggml_tensor * llama_model_hy_v4::graph::build_hc_pre(
llama_model_hy_v4::graph::hc_pre
llama_model_hy_v4::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t nt = x->ne[2];
@@ -202,16 +202,15 @@ ggml_tensor * llama_model_hy_v4::graph::build_hc_pre(
cb(pre, "hc_pre", il);
// post = magnitude*sigmoid(mixes[hc:2hc]*scale_post + base_post) + eps
ggml_tensor * po = hy_v4_view_2d(ctx0, mixes, hc, nt, hc);
po = ggml_mul(ctx0, po, scale_post);
po = ggml_add(ctx0, po, base_post);
po = ggml_sigmoid(ctx0, po);
po = ggml_scale(ctx0, po, hparams.hc_magnitude);
po = ggml_scale_bias(ctx0, po, 1.0f, hparams.dsv4_hc_eps);
*post = po;
cb(po, "hc_post_gate", il);
ggml_tensor * post = hy_v4_view_2d(ctx0, mixes, hc, nt, hc);
post = ggml_mul(ctx0, post, scale_post);
post = ggml_add(ctx0, post, base_post);
post = ggml_sigmoid(ctx0, post);
post = ggml_scale(ctx0, post, hparams.hc_magnitude);
post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps);
cb(post, "hc_post_gate", il);
return hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type);
return { hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type), post };
}
ggml_tensor * llama_model_hy_v4::graph::build_hc_post(
@@ -418,12 +417,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
return ggml_cont(ctx0, ggml_top_k(ctx0, score, n_top_k));
}
ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
llama_model_hy_v4::graph::attn_dsa
llama_model_hy_v4::graph::build_attention_dsa(
const llama_model & model,
llm_graph_input_attn_k_dsa * inp_attn_dsa,
ggml_tensor * cur,
ggml_tensor * inp_pos,
ggml_tensor ** last_top_k,
ggml_tensor * last_top_k,
float kq_scale,
int il) const {
const auto & layer = model.layers[il];
@@ -436,11 +436,12 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur);
qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
ggml_tensor * top_k = last_top_k;
if (hparams.is_indexer_full(il)) {
*last_top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il);
cb(*last_top_k, "top_k", il);
top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il);
cb(top_k, "top_k", il);
}
GGML_ASSERT(*last_top_k != nullptr);
GGML_ASSERT(top_k != nullptr);
ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr);
@@ -480,7 +481,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * attn = build_attn(inp_attn_dsa,
nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, *last_top_k, kq_scale, il);
Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, top_k, kq_scale, il);
cb(attn, "attn_kqv", il);
ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur);
@@ -491,7 +492,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * out = build_lora_mm(layer.wo, attn);
cb(out, "attn_out", il);
return out;
return { out, top_k };
}
llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_params & params) :
@@ -520,23 +521,27 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
cur = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale,
model.layers[il].hc_attn_base, &post, il);
hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale,
model.layers[il].hc_attn_base, il);
cur = attn_hc.cur;
cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
cur = is_dsa
? build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, &last_top_k, kq_scale, il)
: build_attention(model, inp_attn, cur, inp_pos, kq_scale, il);
if (is_dsa) {
attn_dsa attn = build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il);
cur = attn.cur;
last_top_k = attn.top_k;
} else {
cur = build_attention(model, inp_attn, cur, inp_pos, kq_scale, il);
}
inpL = build_hc_post(cur, residual, post, il);
inpL = build_hc_post(cur, residual, attn_hc.post, il);
cb(inpL, "hc_attn_out", il);
residual = inpL;
cur = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale,
model.layers[il].hc_ffn_base, &post, il);
hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale,
model.layers[il].hc_ffn_base, il);
cur = ffn_hc.cur;
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -575,7 +580,7 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param
cb(cur, "ffn_out", il);
}
inpL = build_hc_post(cur, residual, post, il);
inpL = build_hc_post(cur, residual, ffn_hc.post, il);
cb(inpL, "l_out", il);
}
src/models/models.h
+54 -21
View File
@@ -1358,24 +1358,33 @@ struct llama_model_deepseek4 : public llama_model_base {
// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens]
ggml_tensor * build_hc_mean(ggml_tensor * x) const;
// returns the collapsed input and fills the post / comb weights
ggml_tensor * build_hc_pre(
// all outputs of the hc mix builders:
// build_hc_pre - cur is the collapsed input, post / comb feed build_hc_post
// build_hc_mixes - pre is the pre weights, post / comb feed build_hc_post
struct hc_pre {
ggml_tensor * cur;
ggml_tensor * post;
ggml_tensor * comb;
};
struct hc_mixes {
ggml_tensor * pre;
ggml_tensor * post;
ggml_tensor * comb;
};
hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const;
// returns the pre weights and fills post / comb
ggml_tensor * build_hc_mixes(
hc_mixes build_hc_mixes(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const;
ggml_tensor * build_hc_post(
@@ -2263,12 +2272,16 @@ struct llama_model_hy_v4 : public llama_model_base {
// iHC (independent Hyper-Connections): pre reduces the hc streams to one and returns the
// per-stream post gates, post writes the sublayer output back into the streams, head
// collapses the streams before the final norm.
ggml_tensor * build_hc_pre(
struct hc_pre {
ggml_tensor * cur; // the collapsed stream
ggml_tensor * post; // per-stream post gates, feeds build_hc_post
};
hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
int il) const;
ggml_tensor * build_hc_post(
@@ -2301,12 +2314,18 @@ struct llama_model_hy_v4 : public llama_model_base {
ggml_tensor * inp_pos,
int il) const;
ggml_tensor * build_attention_dsa(
// the attention output and the top-k to carry to the following "shared" layers
struct attn_dsa {
ggml_tensor * cur;
ggml_tensor * top_k;
};
attn_dsa build_attention_dsa(
const llama_model & model,
llm_graph_input_attn_k_dsa * inp_attn_dsa,
ggml_tensor * cur,
ggml_tensor * inp_pos,
ggml_tensor ** last_top_k,
ggml_tensor * last_top_k,
float kq_scale,
int il) const;
};
@@ -2730,14 +2749,18 @@ struct llama_model_qwen4exp : public llama_model_base {
llm_build_delta_net_base(params), model(model) {}
// HC replaces every layer norm: residual is [n_embd, hc, n_tokens]
ggml_tensor * build_hc_mix(
struct hc_mix {
ggml_tensor * cur; // the mixed [n_embd, T] stream
ggml_tensor * inject; // [hc, T] scatter weights, null when the inject weights are null
};
hc_mix build_hc_mix(
ggml_tensor * x,
ggml_tensor * w_norm,
ggml_tensor * w_down,
ggml_tensor * w_up,
ggml_tensor * w_inject,
ggml_tensor * w_down_inject, // [w_down | w_inject] in one weight, can be null
ggml_tensor ** inject,
int il);
ggml_tensor * build_hc_combine(
@@ -2969,14 +2992,18 @@ struct llama_model_glm5_next : public llama_model_base {
// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens]
ggml_tensor * build_hc_mean(ggml_tensor * x) const;
// returns the collapsed input and fills the post / comb weights
ggml_tensor * build_hc_pre(
// all outputs of the hc mix builder: cur is the collapsed input, post / comb feed build_hc_post
struct hc_pre {
ggml_tensor * cur;
ggml_tensor * post;
ggml_tensor * comb;
};
hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const;
ggml_tensor * build_hc_post(
@@ -3002,9 +3029,15 @@ struct llama_model_glm5_next : public llama_model_base {
ggml_tensor * build_kpool_select(ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * kq_mask, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_kpool * inp_kpool, int il);
ggml_tensor * build_dsa_layer(ggml_tensor * cur, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il);
// attention output (cur) and the selection to carry to the next shared layer (sel)
struct dsa_out {
ggml_tensor * cur;
ggml_tensor * sel;
};
dsa_out build_dsa_layer(ggml_tensor * cur, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il);
};
src/models/qwen4exp.cpp
+34 -29
View File
@@ -343,15 +343,15 @@ static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) {
}
// Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms.
// Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights.
ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
// build_hc_mix returns the mixed [n_embd, T] stream and the [hc, T] scatter weights.
llama_model_qwen4exp::graph::hc_mix
llama_model_qwen4exp::graph::build_hc_mix(
ggml_tensor * x,
ggml_tensor * w_norm,
ggml_tensor * w_down,
ggml_tensor * w_up,
ggml_tensor * w_inject,
ggml_tensor * w_down_inject,
ggml_tensor ** inject,
int il) {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc * n_embd;
@@ -364,15 +364,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
cb(xn, "hc_norm", il);
ggml_tensor * lo = nullptr;
if (inject && w_down_inject) {
ggml_tensor * inject = nullptr;
if (w_down_inject) {
// one matmul for both: the hc_down rows, then the hc_inject rows
const int64_t n_lo = w_down_inject->ne[1] - hc;
ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn);
// scale the combined rows before the split; both lo and inject use 1/hc
lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc);
lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
*inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
cb(*inject, "hc_inject", il);
lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
cb(inject, "hc_inject", il);
} else {
lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc);
}
@@ -405,12 +406,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
}
cb(mixed, "hc_mixed", il);
if (inject && !w_down_inject) {
*inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
cb(*inject, "hc_inject", il);
if (!w_down_inject && w_inject) {
inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
cb(inject, "hc_inject", il);
}
return mixed;
return { mixed, inject };
}
ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
@@ -504,14 +505,14 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);
}
ggml_tensor * inject = nullptr;
ggml_tensor * cur = build_hc_mix(res_hc,
hc_mix attn_mix = build_hc_mix(res_hc,
model.layers[il].hc_attn_norm,
model.layers[il].hc_attn_down,
model.layers[il].hc_attn_up,
model.layers[il].hc_attn_inject,
model.layers[il].hc_attn_down_inject,
&inject, il);
il);
ggml_tensor * cur = attn_mix.cur;
ggml_build_forward_expand(gf, cur);
@@ -523,28 +524,29 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
// everything below is per token, so drop the rows that produce no output
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inject = ggml_get_rows(ctx0, inject, inp_out_ids);
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
attn_mix.inject = ggml_get_rows(ctx0, attn_mix.inject, inp_out_ids);
res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);
res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids);
res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
}
res_hc = build_hc_combine(res_hc, cur, inject, il);
res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
cur = build_hc_mix(res_hc,
hc_mix ffn_mix = build_hc_mix(res_hc,
model.layers[il].hc_ffn_norm,
model.layers[il].hc_ffn_down,
model.layers[il].hc_ffn_up,
model.layers[il].hc_ffn_inject,
model.layers[il].hc_ffn_down_inject,
&inject, il);
il);
cur = ffn_mix.cur;
cur = build_layer_ffn(cur, il);
cb(cur, "ffn_out", il);
res_hc = build_hc_combine(res_hc, cur, inject, il);
res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
// "l_last" is the layer output name that build_cvec and imatrix look for
cb(res_hc, "l_last", il);
@@ -562,9 +564,10 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
}
// the final mixer is the output norm: there is no separate one
ggml_tensor * cur = build_hc_mix(res_hc,
hc_mix head_mix = build_hc_mix(res_hc,
model.hc_head_norm, model.hc_head_down, model.hc_head_up,
nullptr, nullptr, nullptr, -1);
nullptr, nullptr, -1);
ggml_tensor * cur = head_mix.cur;
cb(cur, "result_norm", -1);
res->t_embd = cur;
@@ -641,14 +644,15 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); // [n_embd, hc, n_tokens]
cb(res_hc, "mtp_eh_proj", il);
ggml_tensor * inject = nullptr;
ggml_tensor * cur = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, &inject, il);
hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il);
ggml_tensor * cur = attn_mix.cur;
cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
res_hc = build_hc_combine(res_hc, cur, inject, il);
res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
cur = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, &inject, il);
hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il);
cur = ffn_mix.cur;
cur = build_layer_ffn(cur, il);
res_hc = build_hc_combine(res_hc, cur, inject, il);
res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
// the next draft step reads this residual as its h
ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens);
@@ -657,9 +661,10 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
cb(res->t_h_nextn, "h_nextn", il);
ggml_build_forward_expand(gf, res->t_h_nextn);
cur = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]),
hc_mix head_mix = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]),
layer.nextn.hc_head_norm, layer.nextn.hc_head_down, layer.nextn.hc_head_up,
nullptr, nullptr, nullptr, il);
nullptr, nullptr, il);
cur = head_mix.cur;
cb(cur, "result_norm", -1);
res->t_embd = cur;
tests/llama-context/cases-kv.cpp
+50 -5
View File
@@ -346,7 +346,7 @@ struct case_state_partial_size : ctx_test {
const char * name() const override { return "ps"; }
const char * description() const override {
return "the partial state is not larger than the full state and, without SWA, does not grow with the tokens";
return "the partial state is not larger than the full state";
}
case_result run(model_run & mr) const override {
@@ -376,10 +376,6 @@ struct case_state_partial_size : ctx_test {
CTX_REQUIRE(partial_a <= full_a && partial_b <= full_b, "the partial state is larger than the full state");
if (llama_model_n_swa(mr.model) == 0) {
CTX_REQUIRE(partial_a == partial_b, "the partial state grows with the tokens");
}
return case_result::pass();
}
};
@@ -718,6 +714,54 @@ struct case_state_restore_fragmented : ctx_test {
}
};
//
// no_alloc
//
// - load a second instance of the model with no_alloc, so the tensor data stays on disk
// - create a context from it, which reserves the compute graphs over the dummy buffers
// - the memory use must still be reported, and nothing real may be allocated for the memory
struct case_noalloc : ctx_test {
const char * name() const override { return "noalloc"; }
const char * description() const override {
return "with no_alloc the memory use is simulated: non-zero breakdown, null buffer bases";
}
case_result run(model_run & mr) const override {
const std::string & path = mr.params.model.path;
CTX_REQUIRE(!path.empty(), "the model path is not known");
llama_model_params mparams = llama_model_default_params();
mparams.no_alloc = true;
mparams.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model_ptr model(llama_model_load_from_file(path.c_str(), mparams));
CTX_REQUIRE(model, "failed to load the model with no_alloc");
auto ctx = init_ctx(model.get(), common_context_params_to_llama(mr.params), 0);
CTX_REQUIRE(ctx, "failed to create the context with no_alloc");
size_t n_ctx_mem = 0;
for (const auto & [_, data] : llama_get_memory_breakdown(ctx.get())) {
n_ctx_mem += data.context;
}
// models without a memory module (BERT, diffusion) have a null memory
if (llama_get_memory(ctx.get()) != nullptr) {
CTX_REQUIRE(n_ctx_mem > 0, "the simulated memory breakdown is zero");
} else {
CTX_REQUIRE(n_ctx_mem == 0, "a model without memory reported context memory");
}
for (ggml_backend_buffer_t buf : llama_memory_get_buffers(llama_get_memory(ctx.get()))) {
CTX_REQUIRE(ggml_backend_buffer_get_base(buf) == nullptr, "a memory buffer was really allocated");
}
return case_result::pass();
}
};
std::vector<ctx_test_ptr> make_kv_cases() {
std::vector<ctx_test_ptr> cases;
@@ -733,6 +777,7 @@ std::vector<ctx_test_ptr> make_kv_cases() {
cases.push_back(std::make_unique<case_state_restore_failure>());
cases.push_back(std::make_unique<case_state_rotation>());
cases.push_back(std::make_unique<case_state_restore_fragmented>());
cases.push_back(std::make_unique<case_noalloc>());
return cases;
}
tests/llama-context/cases-rs.cpp
+34 -9
View File
@@ -7,16 +7,14 @@
#include "common.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <string>
#include <vector>
// roll back multiple sequences, then replay them in a single batch whose per-seq token count
// exceeds n_ubatch: each seq's replay spans several ubatches while its rollback restore is
// still pending. Compared against a reference context that never advanced past the rollback
// point and decodes the identical replay batch.
// roll back multiple sequences, then replay them in a single batch whose per-seq token count exceeds n_ubatch: each
// seq's replay spans several ubatches while its rollback restore is still pending. Compared against a reference context
// that never advanced past the rollback point and decodes the identical replay batch. Only the first replayed token of
// each seq is compared.
struct case_multi_seq_split_replay : ctx_test {
test_group group() const override { return TEST_GROUP_RS; }
@@ -94,12 +92,16 @@ struct case_multi_seq_split_replay : ctx_test {
ok = decode_replay(ctx_roll.get()) && decode_replay(ctx_ref.get());
CTX_REQUIRE(ok, "the multi-seq replay decode failed");
// both contexts decode identical batches, so the logits should match;
// the first replayed token of each seq reads the state right after the rollback, so its
// logits should match; the later tokens are not compared because small state
// differences at the start grow with each replayed token
// random dummy models can still drift up to ~1.7e-5, so the bound is 1e-4
constexpr float nmse_eps = 1e-4f;
logits_diff diff;
for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {
for (uint32_t s = 0; s < n_seqs; ++s) {
const uint32_t i = s*n_replay; // first replayed token of this seq
const float * l_roll = llama_get_logits_ith(ctx_roll.get(), i);
const float * l_ref = llama_get_logits_ith(ctx_ref.get(), i);
if (l_roll == nullptr || l_ref == nullptr) {
@@ -116,6 +118,29 @@ struct case_multi_seq_split_replay : ctx_test {
TLOG_INF("%s: multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff.diff_max, diff.nmse());
// the replay above can leave the 2 contexts in slightly different states, and the tail
// below assumes they start from the same state - so copy the roll state into ref
for (uint32_t s = 0; s < n_seqs && ok; ++s) {
std::vector<uint8_t> state;
ok = ok && get_seq_state(ctx_roll.get(), (llama_seq_id) s, 0, state);
ok = ok && set_seq_state(ctx_ref.get(), (llama_seq_id) s, 0, state);
// do a second round of state save/load to negate any SWA-positioning effects
if (ok) {
std::vector<uint8_t> restored;
ok = ok && get_seq_state(ctx_ref.get(), (llama_seq_id) s, 0, restored);
// what was loaded into ref must come back out bit-identical
CTX_REQUIRE(ok && state == restored,
"the seq " + std::to_string(s) + " state did not round-trip bit-exact (sizes " +
std::to_string(state.size()) + "/" + std::to_string(restored.size()) + ")");
ok = ok && set_seq_state(ctx_roll.get(), (llama_seq_id) s, 0, restored);
}
}
CTX_REQUIRE(ok, "the multi-seq state copy failed");
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
// in ctx_ref only, then compare identical seq-1-only continuations
constexpr uint32_t n_tail = 4;
@@ -148,7 +173,7 @@ struct case_multi_seq_split_replay : ctx_test {
diff_tail.add(l_roll, l_ref, n_vocab);
}
CTX_REQUIRE(ok && diff_tail.nmse() <= nmse_eps,
CTX_REQUIRE(ok && diff_tail.nmse() <= 0.0f,
"a seq-1-only decode leaked the state of seq 0 (nmse " + std::to_string(diff_tail.nmse()) + ")");
TLOG_INF("%s: seq-1-only decode is independent of seq 0 (max diff %g, nmse %g)\n",
tests/test-backend-ops.cpp
+54 -6
View File
@@ -3679,23 +3679,37 @@ struct test_scale : public test_case {
float scale;
float bias;
bool inplace;
int v; // view (1 : non-contiguous a)
std::string vars() override {
return VARS_TO_STR5(type, ne, scale, bias, inplace);
return VARS_TO_STR6(type, ne, scale, bias, inplace, v);
}
test_scale(ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {10, 10, 10, 10},
float scale = 2.0f,
float bias = 0.0f,
bool inplace = false)
: type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {}
bool inplace = false,
int v = 0)
: type(type), ne(ne), scale(scale), bias(bias), inplace(inplace), v(v) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
auto ne_a = ne;
if (v & 1) {
ne_a[0] *= 3;
ne_a[1] *= 2;
ne_a[2] *= 5;
ne_a[3] *= 4;
}
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());
ggml_set_param(a);
ggml_set_name(a, "a");
if (v & 1) {
a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "view_of_a");
}
ggml_tensor * out;
if (inplace) {
out = ggml_scale_bias_inplace(ctx, a, scale, bias);
@@ -7126,7 +7140,7 @@ struct test_concat : public test_case {
const std::array<int64_t, 4> ne_a;
const int64_t ne_b_d;
const int dim;
const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim))
const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim), 1 << 4: transposed b)
std::string vars() override {
return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v);
@@ -7175,6 +7189,13 @@ struct test_concat : public test_case {
b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0);
ggml_set_name(b, "view_of_b");
} else if (v & 16) {
auto ne = ne_b; std::swap(ne[0], ne[1]);
b = ggml_new_tensor(ctx, type, 4, ne.data());
ggml_set_name(b, "b");
b = ggml_transpose(ctx, b);
ggml_set_name(b, "transpose_of_b");
} else {
b = ggml_new_tensor(ctx, type, 4, ne_b.data());
ggml_set_name(b, "b");
@@ -10854,6 +10875,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f));
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f));
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f));
test_cases.emplace_back(new test_silu_back());
@@ -10893,9 +10918,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_l2_norm (GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-12f, false, false));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {4, 1, 65536, 1}, 1e-6f, false, false));
for (uint32_t n : {1, 127, 129, 511, 513, 2816, 4096, 16385}) {
test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {n, 3, 2, 1}));
test_cases.emplace_back(new test_rms_norm_mul_rope({n, 3, 2, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 3, 2, 1}, 1e-6f, false, false, false, false, true));
}
for (ggml_type set_rows_type : { GGML_TYPE_F32, GGML_TYPE_F16 }) {
test_cases.emplace_back(new test_rms_norm_mul_rope({ 256, 1, 1, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type));
test_cases.emplace_back(new test_rms_norm_mul_rope({ 128, 4, 3, 1 }, 1e-6f, false, true, false, GGML_ROPE_TYPE_NORMAL, false, false, set_rows_type));
for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) {
for (int64_t n : {64, 256, 512, 1024}) {
test_cases.emplace_back(new test_rms_norm_mul_rope({n, 4, 3, 1}, 1e-6f, false, true, true, mode, true, true, set_rows_type));
}
}
}
for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) {
@@ -11792,7 +11828,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding
test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global
for (int v : { 0, 1, 2, 3 }) {
for (int v : { 0, 1, 2, 3, 16 }) {
for (int dim : { 0, 1, 2, 3, }) {
test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v));
test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v));
@@ -12520,6 +12556,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
std::vector<std::unique_ptr<test_case>> test_cases;
// delta-net conv input: conv state followed by the transposed qkv of the ubatch
test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16));
// SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
// note: same bytes either way, so a backend that indexes them differently shows it here
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {
@@ -12549,6 +12588,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
// RMS_NORM and fused variants at decode and prefill shapes
for (int64_t n_tokens : {1, 512}) {
test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {4096, n_tokens, 1, 1}));
test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {128, 32*n_tokens, 1, 1}));
test_cases.emplace_back(new test_rms_norm_mul_rope({4096, n_tokens, 1, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NORMAL, true, false));
test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, n_tokens, 1}, 1e-6f, false, false, true, GGML_ROPE_TYPE_NEOX));
test_cases.emplace_back(new test_rms_norm_mul_rope({128, 8, n_tokens, 1}, 1e-6f, false, true, true, GGML_ROPE_TYPE_NEOX));
}
// POOL_1D max over MoE down-projection amax shapes:
// amax is [1, n_expert, n_tokens] after reducing the whole intermediate dim.
for (int64_t n_ff : { 2048, 4096, 8192 }) {
tests/test-model-resolution.cpp
+2250
View File
File diff suppressed because it is too large. Load diff
tools/mtmd/mtmd-cli.cpp
+5 -3
View File
@@ -75,6 +75,7 @@ static void inject_test_response_marker() {
struct mtmd_cli_context {
mtmd::context_ptr ctx_vision;
mtmd_helper_cap mtmd_cap = {};
common_init_result_ptr llama_init;
llama_model * model;
@@ -120,7 +121,8 @@ struct mtmd_cli_context {
init_vision_context(params);
if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get());
if (!mtmd_cap.chat) {
LOG_ERR("Model does not support chat mode\n");
LOG_ERR("Hint: for TTS models, please use llama-tts\n");
exit(1);
@@ -565,7 +567,7 @@ int main(int argc, char ** argv) {
if (g_is_interrupted) return 130;
if (mtmd_support_audio_stream(ctx.ctx_vision.get())) {
if (ctx.mtmd_cap.audio_stream) {
if (params.image.size() != 1) {
LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n");
return 1;
@@ -624,7 +626,7 @@ int main(int argc, char ** argv) {
if (mtmd_support_audio(ctx.ctx_vision.get())) {
LOG("\n /audio <path> load an audio");
}
if (mtmd_helper_support_video(ctx.ctx_vision.get())) {
if (ctx.mtmd_cap.video) {
LOG("\n /video <path> load a video");
}
LOG("\n /clear clear the chat history");
tools/mtmd/mtmd-helper.cpp
+30 -16
View File
@@ -501,15 +501,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
}
bool mtmd_helper_support_video(const mtmd_context * ctx) {
#ifdef MTMD_VIDEO
return mtmd_support_vision(ctx);
#else
GGML_UNUSED(ctx);
return false;
#endif
}
//
// Video input helpers
//
@@ -1033,17 +1024,40 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
#endif
}
bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mtmd_context * mctx) {
mtmd_helper_cap cap = {};
cap.chat = true;
cap.text_gen = true;
if (!mctx) {
return true;
return cap;
}
auto * model = llama_get_model(lctx);
auto * tmpl = llama_model_chat_template(model, nullptr);
auto info = mtmd_gen_audio_get_info(mctx);
cap.vision = mtmd_support_vision(mctx);
cap.audio = mtmd_support_audio(mctx);
cap.audio_stream = mtmd_support_audio_stream(mctx);
#ifdef MTMD_VIDEO
cap.video = cap.vision;
#endif
const auto gen_audio_type = mtmd_gen_audio_get_info(mctx).type;
cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE;
cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS
|| gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS;
// tts-only model cannot be used for chat (no chat template)
bool is_tts_only = info.type != MTMD_GEN_AUDIO_TYPE_NONE && tmpl == nullptr;
cap.chat = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr;
return !is_tts_only;
return cap;
}
bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
return mtmd_helper_get_capabilities(llama_get_model(lctx), mctx).chat;
}
bool mtmd_helper_support_video(const mtmd_context * ctx) {
#ifdef MTMD_VIDEO
return mtmd_support_vision(ctx);
#else
GGML_UNUSED(ctx);
return false;
#endif
}
tools/mtmd/mtmd-helper.h
+25 -5
View File
@@ -45,9 +45,6 @@ MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
// Note: this also call mtmd_log_set() internally
MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data);
// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time).
MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx);
struct mtmd_helper_bitmap_wrapper {
mtmd_bitmap * bitmap;
mtmd_helper_video * video_ctx;
@@ -186,8 +183,31 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
mtmd_bitmap ** out_bitmap,
char ** out_text);
// return true if model can be used for chat
MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx);
struct mtmd_helper_cap {
bool chat; // false for tts-only model (no chat template)
// input capabilities
bool vision;
bool audio;
bool video; // also depending on MTMD_VIDEO build option
// realtime capabilities
bool audio_stream; // realtime transcription, see mtmd_audio_stream_init()
// output capabilities
bool text_gen; // always true for now
bool audio_gen;
bool audio_gen_spk_ref; // accepts a speaker reference audio for voice cloning
};
typedef struct mtmd_helper_cap mtmd_helper_cap;
// mctx can be NULL (text-only model)
MTMD_API struct mtmd_helper_cap mtmd_helper_get_capabilities(const struct llama_model * model, const struct mtmd_context * mctx);
DEPRECATED(MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx),
"use mtmd_helper_get_capabilities() instead");
DEPRECATED(MTMD_API bool mtmd_helper_support_video(const struct mtmd_context * ctx),
"use mtmd_helper_get_capabilities() instead");
//
// Audio generation helpers
tools/server/server-context.cpp
+9 -6
View File
@@ -1012,16 +1012,17 @@ public:
llama_model * model_tgt = nullptr;
mtmd_context * mctx = nullptr;
mtmd_helper_cap mtmd_cap = {};
// note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
bool has_cap_tts() const {
return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
return mtmd_cap.audio_gen;
}
bool has_cap_chat() const {
return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
return mtmd_cap.chat;
}
server_decision_context decision;
@@ -1397,6 +1398,8 @@ private:
}
}
mtmd_cap = mtmd_helper_get_capabilities(model_tgt, mctx);
if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) {
if (params_base.ctx_shift) {
params_base.ctx_shift = false;
@@ -1705,9 +1708,9 @@ private:
/* reasoning_format */ params_base.reasoning_format,
/* chat_template_kwargs */ params_base.default_template_kwargs,
/* tmpls */ std::move(chat_templates),
/* allow_image */ mctx ? mtmd_support_vision(mctx) : false,
/* allow_audio */ mctx ? mtmd_support_audio (mctx) : false,
/* allow_video */ mctx ? mtmd_helper_support_video(mctx) : false,
/* allow_image */ mtmd_cap.vision,
/* allow_audio */ mtmd_cap.audio,
/* allow_video */ mtmd_cap.video,
/* enable_thinking */ enable_thinking,
/* reasoning_budget */ params_base.sampling.reasoning_budget_tokens,
/* reasoning_budget_msg */ params_base.sampling.reasoning_budget_message,
@@ -5127,7 +5130,7 @@ server_context_meta server_context::get_meta() const {
/* has_inp_video */ impl->chat_params.allow_video,
/* has_cap_chat */ impl->has_cap_chat(),
/* has_cap_tts */ impl->has_cap_tts(),
/* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),
/* has_audio_stream */ impl->mtmd_cap.audio_stream,
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
tools/server/server-models.cpp
+1 -1
View File
@@ -533,7 +533,7 @@ void server_model_meta::update_caps(const common_params & base) {
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
const common_gguf_info info = common_get_gguf_info(params.model.path);
// same rule as mtmd_helper_model_can_chat()
// same rule as mtmd_helper_get_capabilities()
const bool can_chat = !gen_audio || info.has_chat_tmpl;
output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);
n_ctx_train = info.n_ctx_train;
tools/ui/README.md
+1 -2
View File
@@ -406,8 +406,7 @@ Hooks are the thin view-layer between components and stores: they own UI concern
| `use-chat-message-edit-context` | Message edit context (content + extras) |
| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine |
| `use-chat-screen-file-upload` | File upload queue + capability validation |
| `use-chat-screen-scroll` | Scroll container binding + navigation guard |
| `use-auto-scroll` | Auto-scroll controller for streaming |
| `use-stick-to-bottom` | Follows new content while the reader sits at the bottom of a scroller |
| `use-marquee-selection` | Shift+click / marquee range selection |
| `use-keyboard-shortcuts` | Global keyboard shortcuts |
| `use-settings-navigation` | Settings section navigation |
tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
+2
View File
@@ -144,6 +144,7 @@
);
const pickers = useChatFormPickers({
canCompact: () => chatStore.canCompact(conversationsStore.activeMessages),
focusInput: refocusInput,
getCaretOffset: () => inputRef?.getCaretOffset(),
getCwd: () => cwd,
@@ -163,6 +164,7 @@
.getServers()
.some((s) => s.enabled && prefs.isServerToolsEnabled(s.id) && s.url.trim());
},
onCompact: () => chatStore.compactConversation(),
openModelSelector: () => chatFormActionsRef?.openModelSelector(),
setCaretOffset: (offset) => inputRef?.setCaretOffset(offset),
setValue: (v) => {
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte
+4 -4
View File
@@ -30,9 +30,9 @@
{#snippet triggerLabel()}
<span class="flex items-center gap-0.75 {reasoning.isOff ? 'text-muted-foreground' : ''}">
{#if reasoning.isOff}
<LightbulbOff class="size-3 max-md:size-4 shrink-0" />
<LightbulbOff class="size-3.25 max-md:size-4 shrink-0" />
{:else}
<Lightbulb class="size-3 max-md:size-4 shrink-0" />
<Lightbulb class="size-3.25 max-md:size-4 shrink-0" />
{/if}
{#if !isDefault}
@@ -102,9 +102,9 @@
<Button
{...props}
aria-label="Reasoning effort"
class="h-auto gap-1 rounded-sm bg-transparent! backdrop-blur-none! {isDefault
class="-mr-px h-6 min-w-6 gap-1 rounded-sm bg-transparent backdrop-blur-none! {isDefault
? 'px-1'
: 'px-1.75!'} py-1 text-xs hover:bg-transparent!"
: 'px-1.75!'} py-0 text-xs hover:bg-muted-foreground/15 data-[state=open]:bg-muted-foreground/15"
variant="ghost"
>
{@render triggerLabel()}
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte
+46 -1
View File
@@ -8,9 +8,11 @@
gaugePopup,
gaugePopupClose
} from './gauge-popup.svelte';
import { FoldVertical, Loader2 } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as Drawer from '$lib/components/ui/drawer';
import { useContextGauge } from '$lib/hooks/use-context-gauge.svelte';
import { deviceStore } from '$lib/stores';
import { agenticStore, chatStore, conversationsStore, deviceStore } from '$lib/stores';
import { formatParameters } from '$lib/utils/formatters';
const gauge = useContextGauge();
@@ -53,6 +55,25 @@
gauge.contextTotal > 0 &&
(gauge.activeModelId !== null || gauge.isActiveModelLoaded)
);
const showCompactButton = $derived(chatStore.canCompact(conversationsStore.activeMessages));
const activeConvId = $derived(conversationsStore.activeConversation?.id ?? null);
// A summary in flight keeps the card open for the numbers, so the action
// reports the run instead of inviting a second one.
const isCompacting = $derived(activeConvId !== null && chatStore.isCompacting(activeConvId));
const isCompactDisabled = $derived.by(() => {
if (!activeConvId) return true;
return chatStore.isChatLoading(activeConvId) || agenticStore.isRunning(activeConvId);
});
function handleCompact() {
gaugePopupClose();
chatStore.compactConversation();
}
</script>
{#snippet usage()}
@@ -62,6 +83,26 @@
</span>
{/snippet}
{#snippet compactAction()}
{#if showCompactButton}
<Button
class="my-1 mt-3 w-full gap-1.5"
disabled={isCompactDisabled}
onclick={handleCompact}
size="sm"
variant="tertiary"
>
{#if isCompacting}
<Loader2 class="h-3.5 w-3.5 animate-spin" />
Compacting...
{:else}
<FoldVertical class="h-3.5 w-3.5" />
Compact conversation
{/if}
</Button>
{/if}
{/snippet}
{#snippet body()}
{#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded}
<ContextGaugeLoadModel
@@ -123,6 +164,8 @@
{@render usage()}
{@render body()}
{@render compactAction()}
</div>
</Drawer.Content>
</Drawer.Root>
@@ -145,6 +188,8 @@
</div>
{@render body()}
{@render compactAction()}
</div>
</div>
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte
+9 -1
View File
@@ -2,6 +2,7 @@
import { goto } from '$app/navigation';
import {
ChatMessageAssistant,
ChatMessageCompaction,
ChatMessageMcpPrompt,
ChatMessageSynthetic,
ChatMessageSystem,
@@ -14,7 +15,7 @@
SYSTEM_MESSAGE_PLACEHOLDER
} from '$lib/constants';
import { setChatMessageActionsContext, setChatMessageEditContext } from '$lib/contexts';
import { AgenticSectionType, AttachmentType, MessageRole } from '$lib/enums';
import { AgenticSectionType, AttachmentType, MessageRole, MessageType } from '$lib/enums';
import { DatabaseService } from '$lib/services/database.service';
import { chatStore, conversationsStore, deviceStore } from '$lib/stores';
import type {
@@ -60,6 +61,7 @@
// Synthetic cwd-change messages render with the folder-row UI instead
// of a user bubble. The persisted flag is the single source of truth.
let isSynthetic = $derived(Boolean(message.isSynthetic));
let isCompaction = $derived(message.type === MessageType.COMPACTION);
let rawEditContent = $derived.by(() => {
if (message.role !== MessageRole.ASSISTANT) return undefined;
@@ -313,6 +315,10 @@
chatActions.regenerateWithBranching(message, modelOverride);
}
function handleRegenerateCompaction() {
chatActions.regenerateCompaction(message);
}
function handleContinue() {
chatActions.continueAssistantMessage(message);
}
@@ -409,6 +415,8 @@
<ChatMessageSystem bind:textareaElement class={className} {message} />
{:else if mcpPromptExtra}
<ChatMessageMcpPrompt class={className} mcpPrompt={mcpPromptExtra} {message} />
{:else if isCompaction}
<ChatMessageCompaction class={className} {message} onRegenerate={handleRegenerateCompaction} />
{:else if isSynthetic}
<ChatMessageSynthetic class={className} {message} />
{:else if message.role === MessageRole.USER}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte
+5 -1
View File
@@ -52,6 +52,9 @@
let isStreaming = $derived(chatStore.isStreaming());
let hasNoContent = $derived(!message?.content?.trim());
let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming);
// A compaction summary reports its own progress on the compaction row, so
// this turn stays quiet while it runs instead of duplicating the indicator.
let isCompacting = $derived(chatStore.isCompacting(message.convId));
// during a router auto-load the message has no model yet: target the model frozen in the
// persisted stream state (survives a reload), then fall back to the dropdown selection
@@ -66,6 +69,7 @@
let showProcessingInfoTop = $derived(
message?.role === MessageRole.ASSISTANT &&
isActivelyProcessing &&
!isCompacting &&
hasNoContent &&
!isAgentic &&
isLastAssistantMessage
@@ -74,6 +78,7 @@
let showProcessingInfoBottom = $derived(
message?.role === MessageRole.ASSISTANT &&
isActivelyProcessing &&
!isCompacting &&
(!hasNoContent || isAgentic) &&
isLastAssistantMessage
);
@@ -212,7 +217,6 @@
onRawOutputToggle={(enabled) => (showRawOutput = enabled)}
{onRegenerate}
rawOutputEnabled={showRawOutput}
role={MessageRole.ASSISTANT}
showRawOutputSwitch={currentConfig.showRawOutputSwitch}
/>
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte
+9
View File
@@ -31,6 +31,15 @@
promptMs={liveLlm.prompt_ms}
promptTokens={liveLlm.prompt_n}
/>
{:else if showMessageStats && message.timings?.compaction}
{@const summary = message.timings.compaction}
<ChatMessageStatistics
mode={ChatMessageStatisticsMode.GENERATION}
predictedMs={summary.predicted_ms}
predictedTokens={summary.predicted_n}
promptMs={summary.prompt_ms}
promptTokens={summary.prompt_n}
/>
{:else if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms}
{@const agentic = message.timings.agentic}
<ChatMessageStatistics
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageCompaction.svelte
+156
View File
@@ -0,0 +1,156 @@
<script lang="ts">
import { FoldVertical, Loader2 } from '@lucide/svelte';
import {
ChatMessageActionIcons,
ChatMessageAssistantStatistics,
ChatMessageEditForm,
CollapsibleContentBlock,
MarkdownContent
} from '$lib/components/app';
import { COLLAPSIBLE_TRANSITION_MS } from '$lib/constants';
import { getChatMessageEditContext } from '$lib/contexts';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { chatStore, conversationsStore, settingsStore } from '$lib/stores';
import type { DatabaseMessage } from '$lib/types';
import { compactionTokensBefore, formatNumber } from '$lib/utils';
interface Props {
class?: string;
message: DatabaseMessage;
onRegenerate: () => void;
}
let { class: className = '', message, onRegenerate }: Props = $props();
const COMPACTION_HEADER = 'Conversation compacted';
const COMPACTION_RUNNING_LABEL = 'Compacting conversation';
const COMPACTION_GENERATING_LABEL = 'Generating summary';
// The shared row context, so the summary is editable and branchable through
// the same actions as a chat turn.
const editCtx = getChatMessageEditContext();
const currentConfig = $derived(settingsStore.config);
const hasContent = $derived(message.content.trim().length > 0);
// Only the node a running summary writes to reports progress: a finished
// compaction further up the branch keeps its headline while another one runs.
const isSummarizing = $derived(chatStore.getCompactingNodeId(message.convId) === message.id);
const processingState = useProcessingState();
// The summary request feeds the shared processing state, so the body shows
// the same prompt progress line a chat turn does while it waits.
$effect(() => {
if (isSummarizing) processingState.startMonitoring();
});
// The header carries the run: the prompt progress until the summary starts
// streaming, the phase after that. There is nothing to collapse before the
// first summary token arrives.
const runningLabel = $derived.by(() => {
// The prompt is through: whatever streams in now is the summary itself.
if (hasContent) return `${COMPACTION_RUNNING_LABEL} — ${COMPACTION_GENERATING_LABEL}`;
const progress = processingState.getPromptProgressText();
return progress
? `${COMPACTION_RUNNING_LABEL} — ${progress}`
: `${COMPACTION_RUNNING_LABEL}...`;
});
// Shrinking the branch is the feature's own headline, so the finished
// toggle reports the context size it compacted.
const tokensBefore = $derived(compactionTokensBefore(conversationsStore.activeMessages, message));
const completedLabel = $derived(
tokensBefore === null
? COMPACTION_HEADER
: `${COMPACTION_HEADER} from ${formatNumber(tokensBefore)}`
);
const title = $derived(isSummarizing || !hasContent ? runningLabel : completedLabel);
let open = $state(false);
// A toggle by hand outranks the in-progress expansion, the way the agentic
// sections treat their own overrides.
let userChoseOpen = $state<boolean | null>(null);
// The progress lives in the body (as it does for an assistant turn), so the
// row opens while the summary runs and collapses once it is done.
$effect(() => {
if (userChoseOpen !== null) return;
open = isSummarizing;
});
let rootEl = $state<HTMLDivElement>();
// The scroll waiting for the body to reach its height, so closing the block
// before that leaves no scroll behind it.
let settleTimer: ReturnType<typeof setTimeout> | null = null;
/**
* A toggle hands the reader the block's own boundary, just below the tabs: one
* scroll, and only once the body has its height - before that the page is too
* short to scroll the divider up there and the call would clamp.
*/
function handleToggle() {
userChoseOpen = open;
cancelSettledScroll();
if (!rootEl) return;
settleTimer = setTimeout(() => {
settleTimer = null;
rootEl?.scrollIntoView({ behavior: 'smooth', block: 'start' });
}, COLLAPSIBLE_TRANSITION_MS);
}
function cancelSettledScroll() {
if (settleTimer === null) return;
clearTimeout(settleTimer);
settleTimer = null;
}
$effect(() => cancelSettledScroll);
</script>
<div bind:this={rootEl} class="scroll-mt-[calc(var(--chat-tabs-height,0px)+1rem)] {className}">
{#if editCtx.isEditing}
<ChatMessageEditForm />
{:else}
<CollapsibleContentBlock
bind:open
class="my-2"
collapsible={hasContent}
flushBody
icon={isSummarizing ? Loader2 : FoldVertical}
iconClass={isSummarizing ? 'h-3.5 w-3.5 animate-spin' : 'h-3.5 w-3.5'}
onToggle={handleToggle}
shimmerTitle={isSummarizing || !hasContent}
{title}
variant="divider"
>
<MarkdownContent class="text-muted-foreground" content={message.content} />
<!-- The summary's own statistics live with the summary, so the divider
stays a one-line headline. Only the conversation model writes the
summary, so there is no selector to swap it with. -->
<div class="info my-6 grid gap-4 tabular-nums">
<ChatMessageAssistantStatistics
isLoading={chatStore.isLoading}
{message}
{processingState}
showMessageStats={currentConfig.showMessageStats}
/>
</div>
{#if hasContent && !isSummarizing}
<!-- Copy, edit, fork, regenerate and delete all apply to the summary:
regenerate re-runs it in place. Continue stays a chat-turn flow. -->
<ChatMessageActionIcons actionsPosition="left" justify="start" {onRegenerate} />
{/if}
</CollapsibleContentBlock>
{/if}
</div>
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte
+2 -2
View File
@@ -5,7 +5,7 @@
ChatMessageMcpPromptContent
} from '$lib/components/app';
import { getChatMessageEditContext } from '$lib/contexts';
import { McpPromptVariant, MessageRole } from '$lib/enums';
import { McpPromptVariant } from '$lib/enums';
import type { DatabaseMessageExtraMcpPrompt } from '$lib/types';
interface Props {
@@ -36,7 +36,7 @@
{#if message.timestamp}
<div class="max-w-[80%]">
<ChatMessageActionIcons actionsPosition="right" justify="end" role={MessageRole.USER} />
<ChatMessageActionIcons actionsPosition="right" justify="end" />
</div>
{/if}
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte
+2 -2
View File
@@ -5,7 +5,7 @@
import { Card } from '$lib/components/ui/card';
import { INPUT_CLASSES } from '$lib/constants';
import { getChatMessageEditContext } from '$lib/contexts';
import { KeyboardKey, MessageRole } from '$lib/enums';
import { KeyboardKey } from '$lib/enums';
import { settingsStore } from '$lib/stores';
import { autoResizeTextarea, isIMEComposing } from '$lib/utils';
@@ -191,7 +191,7 @@
{#if message.timestamp}
<div class="max-w-[80%]">
<ChatMessageActionIcons actionsPosition="right" justify="end" role={MessageRole.USER} />
<ChatMessageActionIcons actionsPosition="right" justify="end" />
</div>
{/if}
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUser.svelte
+7 -3
View File
@@ -6,7 +6,7 @@
ChatMessageUserBubble
} from '$lib/components/app/chat';
import { getChatMessageEditContext } from '$lib/contexts';
import { ChatMessageStatisticsMode, MessageRole } from '$lib/enums';
import { ChatMessageStatisticsMode } from '$lib/enums';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { chatStore, settingsStore } from '$lib/stores';
@@ -29,7 +29,11 @@
const processingState = useProcessingState();
const currentConfig = $derived(settingsStore.config);
const isActivelyProcessing = $derived(isLastUserMessage && chatStore.isLoading);
// A compaction summary reports its own progress on the compaction row, so
// the summarized turn does not show live reading stats while it runs.
const isActivelyProcessing = $derived(
isLastUserMessage && chatStore.isLoading && !chatStore.isCompacting(message.convId)
);
// For agentic turns, prefer the cumulative agentic.llm totals over per-call timings.
let storedReadingStats = $derived.by(() => {
@@ -107,7 +111,7 @@
{#if message.timestamp}
<div class="max-w-[80%]">
<ChatMessageActionIcons actionsPosition="right" justify="end" role={MessageRole.USER} />
<ChatMessageActionIcons actionsPosition="right" justify="end" />
</div>
{/if}
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionIcons/ChatMessageActionIcons.svelte
+2 -5
View File
@@ -10,11 +10,9 @@
import Label from '$lib/components/ui/label/label.svelte';
import { Switch } from '$lib/components/ui/switch';
import { getChatMessageActionsContext, getChatMessageEditContext } from '$lib/contexts';
import { MessageRole } from '$lib/enums';
import { conversationsStore } from '$lib/stores';
interface Props {
role: MessageRole.USER | MessageRole.ASSISTANT;
justify: 'start' | 'end';
actionsPosition: 'left' | 'right';
onRegenerate?: () => void;
@@ -31,7 +29,6 @@
onRawOutputToggle,
onRegenerate,
rawOutputEnabled = false,
role,
showRawOutputSwitch = false
}: Props = $props();
@@ -81,11 +78,11 @@
<ActionIcon icon={Edit} onclick={editCtx.startEdit} tooltip="Edit" />
{#if role === MessageRole.ASSISTANT && onRegenerate}
{#if onRegenerate}
<ActionIcon icon={RefreshCw} onclick={() => onRegenerate()} tooltip="Regenerate" />
{/if}
{#if role === MessageRole.ASSISTANT && onContinue}
{#if onContinue}
<ActionIcon icon={ArrowRight} onclick={onContinue} tooltip="Continue" />
{/if}
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte
+6 -7
View File
@@ -14,11 +14,10 @@
interface Props {
messages?: DatabaseMessage[];
onUserAction?: () => void;
onMessagesReady?: (messageCount: number) => void;
}
let { messages = [], onMessagesReady, onUserAction }: Props = $props();
let { messages = [], onMessagesReady }: Props = $props();
let allConversationMessages = $state<DatabaseMessage[]>([]);
@@ -26,7 +25,6 @@
const chatActions: ChatMessageActions = {
continueAssistantMessage: async (message: DatabaseMessage) => {
onUserAction?.();
await chatStore.continueAssistantMessage(message.id);
refreshAllMessages();
},
@@ -52,7 +50,6 @@
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => {
onUserAction?.();
// in-place edit: the store already updated activeMessages and no
// branch is created, so sibling info stays valid without a refetch
await chatStore.editUserMessagePreserveResponses(message.id, newContent, newExtras);
@@ -63,7 +60,6 @@
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => {
onUserAction?.();
await chatStore.editMessageWithBranching(message.id, newContent, newExtras);
refreshAllMessages();
},
@@ -73,7 +69,6 @@
newContent: string,
shouldBranch: boolean
) => {
onUserAction?.();
await chatStore.editAssistantMessage(message.id, newContent, shouldBranch);
// only a branch changes sibling info; an in-place edit already
@@ -92,8 +87,12 @@
await conversationsStore.navigateToSibling(siblingId);
},
regenerateCompaction: async (message: DatabaseMessage) => {
await chatStore.regenerateCompaction(message.id);
refreshAllMessages();
},
regenerateWithBranching: async (message: DatabaseMessage, modelOverride?: string) => {
onUserAction?.();
await chatStore.regenerateMessageWithBranching(message.id, modelOverride);
refreshAllMessages();
}
tools/ui/src/lib/components/app/chat/ChatMessages/LazyChatMessage.svelte
+3 -1
View File
@@ -1,5 +1,6 @@
<script lang="ts">
import ChatMessage from './ChatMessage/ChatMessage.svelte';
import { MessageType } from '$lib/enums';
import { chatStore } from '$lib/stores';
import type { ChatMessageActions } from '$lib/types';
@@ -78,7 +79,8 @@
<div
bind:this={wrapperEl}
class:chat-message--eager={eager}
class:chat-message--synthetic={Boolean(message.isSynthetic)}
class:chat-message--synthetic={Boolean(message.isSynthetic) &&
message.type !== MessageType.COMPACTION}
class="chat-message"
>
{#if shouldMount}
tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
+18 -129
View File
@@ -21,12 +21,11 @@
CHAT_FORM_MOVE_TO_BOTTOM_OUT_MS,
ROUTES
} from '$lib/constants';
import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
import { useChatScreenActiveModel } from '$lib/hooks/use-chat-screen-active-model.svelte';
import { useChatScreenDragAndDrop } from '$lib/hooks/use-chat-screen-drag-and-drop.svelte';
import { useChatScreenFileUpload } from '$lib/hooks/use-chat-screen-file-upload.svelte';
import { useChatScreenScroll } from '$lib/hooks/use-chat-screen-scroll.svelte';
import { useKeyboardShortcuts } from '$lib/hooks/use-keyboard-shortcuts.svelte';
import { createStickToBottom } from '$lib/hooks/use-stick-to-bottom.svelte';
import {
chatStore,
conversationsStore,
@@ -35,16 +34,10 @@
settingsStore
} from '$lib/stores';
import { parseFilesToMessageExtras } from '$lib/utils/browser-only';
import { onDestroy, onMount, tick, untrack } from 'svelte';
import { tick, untrack } from 'svelte';
let { showCenteredEmpty = false } = $props();
let disableAutoScroll = $derived(
Boolean(settingsStore.config.disableAutoScroll) || deviceStore.isMobile
);
let isMobileUserScrolledUp = $state(false);
let mobileScrollDownHint = $state(false);
let mobileScrollDownHintLockedUntil = $state(0);
let emptyFileNames = $state<string[]>([]);
let showDeleteDialog = $state(false);
let showEmptyFileDialog = $state(false);
@@ -69,8 +62,9 @@
return '0.5rem';
});
const autoScroll = createAutoScrollController();
const scroll = useChatScreenScroll(autoScroll);
// The page is the scroller: it follows new content while the reader sits at
// the bottom, and otherwise leaves their position alone.
const stick = createStickToBottom();
const activeModel = useChatScreenActiveModel();
const fileUpload = useChatScreenFileUpload({
activeModelId: () => activeModel.activeModelId,
@@ -91,19 +85,6 @@
}
});
function handleMobileScroll() {
if (!deviceStore.isMobile) return;
const container = scroll.chatScrollContainer;
if (!container) return;
const distanceFromBottom =
container.scrollHeight - container.clientHeight - container.scrollTop;
isMobileUserScrolledUp = distanceFromBottom > 300;
}
async function handleDeleteConfirm() {
const conversation = conversationsStore.activeConversation;
@@ -135,7 +116,8 @@
return false;
}
handleSendLikeScroll();
// Sending re-arms following: the reply streams in under the message.
stick.scrollToBottom();
await chatStore.sendMessage(message, result?.extras);
@@ -144,12 +126,9 @@
let lastScrolledConversationId: string | null = null;
// Lands at the bottom of a conversation the first time its messages
// render, whether the route comes from another conversation, from a
// non-conversation route or from a page refresh. Runs before the first
// paint of the new rows, and the landing pin keeps the bottom in place
// while the rows settle. Independent of the auto-scroll setting, which
// only governs following a streaming response.
// Lands at the bottom of a conversation the first time its messages render.
// The resize observer keeps it there while the list settles (lazy rows,
// syntax highlighting, images) without polling for a stable height.
async function handleMessagesReady(messageCount: number) {
if (messageCount === 0) return;
@@ -159,10 +138,7 @@
lastScrolledConversationId = id;
await tick();
if (conversationsStore.activeConversation?.id !== id) return;
scroll.landAtBottom();
stick.scrollToBottom();
}
/**
@@ -213,53 +189,6 @@
if (isEmpty !== isFormCentered) untrack(() => void moveForm());
});
function handleSendLikeScroll() {
if (!deviceStore.isMobile) {
autoScroll.enable();
}
setTimeout(() => {
const container = scroll.chatScrollContainer;
if (!container) return;
// the bubble of the message just sent: the last user bubble, whether or
// not the answer row below it has rendered yet
const userBubbles = container.querySelectorAll<HTMLElement>(
'.chat-message-user .chat-message-user-bubble'
);
const lastUserBubble = userBubbles[userBubbles.length - 1] ?? null;
if (deviceStore.isMobile) {
// Keep the last user message bubble just above the input on mobile
const bubbleHeight = lastUserBubble?.scrollHeight ?? 0;
const baseHeight = container.scrollHeight - innerHeight;
container.scrollTo({
behavior: 'smooth',
top: bubbleHeight > 0 ? baseHeight - bubbleHeight : baseHeight
});
} else if (lastUserBubble) {
// On desktop, place the last user message near the top of the viewport
const topPadding = 24;
const bubbleRect = lastUserBubble.getBoundingClientRect();
container.scrollTo({
behavior: 'smooth',
top: Math.max(0, container.scrollTop + bubbleRect.top - topPadding)
});
} else {
autoScroll.scrollToBottom();
}
}, 100);
if (deviceStore.isMobile) {
autoScroll.setDisabled(disableAutoScroll);
mobileScrollDownHint = true;
mobileScrollDownHintLockedUntil = Date.now() + 500;
}
}
function handleErrorDialogOpenChange(open: boolean) {
if (!open) {
chatStore.dismissErrorDialog();
@@ -271,50 +200,21 @@
}
$effect(() => {
const shouldDisableAutoScroll =
settingsStore.config.disableAutoScroll ||
(deviceStore.isMobile && isCurrentConversationLoading);
autoScroll.setDisabled(shouldDisableAutoScroll);
if (!shouldDisableAutoScroll) {
autoScroll.enable();
}
stick.setEnabled(!settingsStore.config.disableAutoScroll);
});
onMount(() => {
autoScroll.startObserving();
$effect(() => {
stick.attach(document.documentElement, document.body);
if (!disableAutoScroll) {
autoScroll.enable();
}
if (deviceStore.isMobile && isCurrentConversationLoading) {
mobileScrollDownHint = true;
mobileScrollDownHintLockedUntil = Date.now() + 500;
}
handleMobileScroll();
return () => stick.detach();
});
onDestroy(() => autoScroll.destroy());
</script>
{#if dragAndDrop.isDragOver}
<ChatScreenDragOverlay />
{/if}
<svelte:window
onkeydown={handleKeydown}
onscroll={(e) => {
scroll.handleScroll(e);
handleMobileScroll();
if (e.isTrusted && Date.now() > mobileScrollDownHintLockedUntil) {
mobileScrollDownHint = false;
}
}}
/>
<svelte:window onkeydown={handleKeydown} onscroll={() => stick.handleScroll()} />
{#if isServerLoading}
<ServerLoadingSplash />
@@ -332,9 +232,6 @@
<ChatMessages
messages={conversationsStore.activeMessages}
onMessagesReady={handleMessagesReady}
onUserAction={() => {
handleSendLikeScroll();
}}
/>
{/if}
@@ -367,16 +264,8 @@
{/if}
<div class="pointer-events-none flex flex-col gap-6 items-center w-full">
{#if (deviceStore.isMobile ? mobileScrollDownHint || isMobileUserScrolledUp : autoScroll.userScrolledUp) && page.url.hash.includes(ROUTES.CHAT) && page.params.id}
<ChatScreenActionScrollDown
onclick={() => {
mobileScrollDownHint = false;
scroll.chatScrollContainer?.scrollTo({
behavior: 'smooth',
top: scroll.chatScrollContainer.scrollHeight
});
}}
/>
{#if !stick.atBottom && page.url.hash.includes(ROUTES.CHAT) && page.params.id}
<ChatScreenActionScrollDown onclick={() => stick.scrollToBottom('smooth')} />
{/if}
</div>
tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenDialogsAndAlerts.svelte
+2 -9
View File
@@ -1,8 +1,7 @@
<script lang="ts">
import { Trash2 } from '@lucide/svelte';
import {
DialogChatError,
DialogConfirmation,
DialogConversationDelete,
DialogEmptyFileAlert,
DialogFileUploadError
} from '$lib/components/app';
@@ -24,16 +23,10 @@
fileErrorData={fileUpload.fileErrorData}
/>
<DialogConfirmation
<DialogConversationDelete
bind:open={showDeleteDialog}
cancelText="Cancel"
confirmText="Delete"
description="Are you sure you want to delete this conversation? This action cannot be undone and will permanently remove all messages in this conversation."
icon={Trash2}
onCancel={() => (showDeleteDialog = false)}
onConfirm={handleDeleteConfirm}
title="Delete Conversation"
variant="destructive"
/>
<DialogEmptyFileAlert
tools/ui/src/lib/components/app/chat/index.ts
+9 -5
View File
@@ -412,10 +412,7 @@ export { default as ChatFormPickers } from './ChatForm/ChatFormPickers/ChatFormP
*
* @example
* ```svelte
* <ChatMessages
* messages={activeMessages()}
* onUserAction={resetAutoScroll}
* />
* <ChatMessages messages={activeMessages()} onMessagesReady={handleReady} />
* ```
*/
export { default as ChatMessages } from './ChatMessages/ChatMessages.svelte';
@@ -549,6 +546,13 @@ export { default as ChatMessageCwdChange } from './ChatMessages/ChatMessage/Chat
*/
export { default as ChatMessageSynthetic } from './ChatMessages/ChatMessage/ChatMessageSynthetic.svelte';
/**
* Collapsible summary card for compaction nodes. Shows a shimmering pending
* header while the summary streams in, then the stored summary as markdown;
* later requests slice the branch at this node.
*/
export { default as ChatMessageCompaction } from './ChatMessages/ChatMessage/ChatMessageCompaction.svelte';
/**
* Formatted content display for MCP prompt messages. Renders the full prompt
* content with arguments in a readable format. Used within ChatMessageMcpPrompt
@@ -616,7 +620,7 @@ export { default as ChatMessageSystem } from './ChatMessages/ChatMessage/ChatMes
*
* **Architecture:**
* - Composes ChatMessages, ChatScreenForm, and dialogs
* - Manages auto-scroll via `createAutoScrollController()` hook
* - Follows new content via `createStickToBottom()` while the reader is at the bottom
* - Handles file upload pipeline (validation → processing → state update)
* - Integrates with serverStore for loading/error/warning states
* - Tracks active model for modality validation (vision, audio)
tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte
+124 -53
View File
@@ -1,5 +1,6 @@
<script lang="ts">
import ChevronDown from '@lucide/svelte/icons/chevron-down';
import CollapsibleRegion from '$lib/components/app/misc/CollapsibleRegion.svelte';
import * as Collapsible from '$lib/components/ui/collapsible/index.js';
import { cn } from '$lib/components/ui/utils';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
@@ -16,6 +17,21 @@
titleSnippet?: Snippet;
subtitle?: string;
shimmerTitle?: boolean;
/**
* `divider` centers the toggle between two rules, for blocks that mark a
* section boundary in the conversation instead of decorating a message.
*/
variant?: 'default' | 'divider';
/**
* Render the header only, without a chevron or a body, for a block that
* has nothing to show yet and turns into a collapsible once it has.
*/
collapsible?: boolean;
/**
* Drop the reasoning-style rule and indent, for a body that spans the
* message width on its own.
*/
flushBody?: boolean;
onToggle?: () => void;
children: Snippet;
}
@@ -23,6 +39,8 @@
let {
children,
class: className = '',
collapsible = true,
flushBody = false,
icon: IconComponent,
iconClass = ICON_CLASS_DEFAULT,
iconUrl = null,
@@ -31,7 +49,8 @@
shimmerTitle = false,
subtitle,
title = '',
titleSnippet
titleSnippet,
variant = 'default'
}: Props = $props();
function hideBrokenIcon(event: Event) {
@@ -39,6 +58,60 @@
}
</script>
{#snippet toggleIcon(layoutClass: string)}
{#if iconUrl}
<img
alt=""
class={cn('shrink-0 rounded-sm', layoutClass, iconClass)}
onerror={hideBrokenIcon}
src={iconUrl}
/>
{:else if IconComponent}
<IconComponent class={cn('shrink-0 text-muted-foreground/60', layoutClass, iconClass)} />
{/if}
{/snippet}
{#snippet dividerRow()}
<span aria-hidden="true" class="h-px min-w-4 flex-1 bg-current opacity-25"></span>
<div class="flex min-w-0 items-center gap-1.5">
{@render toggleIcon('')}
{@render toggleTitle('text-xs whitespace-nowrap')}
{#if collapsible}
<ChevronDown
class={cn(
'size-3.5 shrink-0 text-muted-foreground/60 transition-transform duration-150 ease-out',
open && 'rotate-180'
)}
/>
{/if}
</div>
<span aria-hidden="true" class="h-px min-w-4 flex-1 bg-current opacity-25"></span>
{/snippet}
{#snippet toggleTitle(layoutClass: string)}
<span
class={cn(
'min-w-0 font-medium',
layoutClass,
shimmerTitle ? 'shimmer-text' : 'text-foreground/80'
)}
>
{#if titleSnippet}
{@render titleSnippet()}
{:else}
{title}
{/if}
</span>
{#if subtitle}
<span class="text-xs italic text-muted-foreground/70">{subtitle}</span>
{/if}
{/snippet}
<Collapsible.Root
class={cn('group/collapsible', 'my-0!', className)}
onOpenChange={(value) => {
@@ -47,62 +120,60 @@
}}
{open}
>
<Collapsible.Trigger
class={cn(
'flex w-full cursor-pointer items-start justify-between gap-2 text-left',
'py-1.5 pr-1'
)}
>
<div class="flex min-w-0 items-start gap-2 text-muted-foreground">
{#if iconUrl}
<img
alt=""
class={cn('shrink-0 rounded-sm mt-0.75', iconClass)}
onerror={hideBrokenIcon}
src={iconUrl}
/>
{:else if IconComponent}
<IconComponent class={cn('shrink-0 text-muted-foreground/60 mt-0.75', iconClass)} />
{/if}
<span
class={cn(
'min-w-0 overflow-x-auto text-sm font-medium',
shimmerTitle ? 'shimmer-text' : 'text-foreground/80'
)}
>
{#if titleSnippet}
{@render titleSnippet()}
{:else}
{title}
{/if}
</span>
{#if subtitle}
<span class="text-xs italic text-muted-foreground/70">{subtitle}</span>
{/if}
{#if variant === 'divider' && !collapsible}
<div class="flex w-full items-center gap-2 py-1.5 text-muted-foreground">
{@render dividerRow()}
</div>
{:else if variant === 'divider'}
<Collapsible.Trigger
class="flex w-full cursor-pointer items-center gap-2 py-1.5 text-muted-foreground"
>
{@render dividerRow()}
<ChevronDown
<span class="sr-only">Toggle content</span>
</Collapsible.Trigger>
{:else}
<Collapsible.Trigger
class={cn(
'size-4 shrink-0 text-muted-foreground/60 transition-all duration-150 ease-out opacity-0 group-hover/collapsible:opacity-100 mt-0.75',
open && 'rotate-180'
'flex w-full cursor-pointer items-start justify-between gap-2 text-left',
'py-1.5 pr-1'
)}
/>
>
<div class="flex min-w-0 items-start gap-2 text-muted-foreground">
{@render toggleIcon('mt-0.75')}
<span class="sr-only">Toggle content</span>
</Collapsible.Trigger>
<Collapsible.Content>
<!-- Collapsible.Content renders its children unconditionally and only sets
`hidden`, so a closed block would keep re-rendering its whole body on
every streamed token. Gate on `open` so collapsed content costs nothing. -->
{#if open}
<div class="pl-1.5 grid min-w-0" style="min-height: var(--min-message-height);">
<div class="min-w-0 border-l border-muted-foreground/20 pl-4 pb-2 my-2">
{@render children()}
</div>
{@render toggleTitle('overflow-x-auto text-sm')}
</div>
{/if}
</Collapsible.Content>
<ChevronDown
class={cn(
'size-4 shrink-0 text-muted-foreground/60 transition-all duration-150 ease-out opacity-0 group-hover/collapsible:opacity-100 mt-0.75',
open && 'rotate-180'
)}
/>
<span class="sr-only">Toggle content</span>
</Collapsible.Trigger>
{/if}
{#if collapsible}
<!-- Content is force-mounted so the trigger keeps a real `aria-controls` target;
the region owns opening and closing. It animates the height instead of
snapping it, and unmounts the body once the collapse is over - a closed
block still costs nothing, which matters while tokens stream past it. -->
<Collapsible.Content forceMount>
<CollapsibleRegion {open}>
<div class="pl-1.5 grid min-w-0" style="min-height: var(--min-message-height);">
<div
class={cn(
'min-w-0 pb-2 my-2',
!flushBody && 'border-l border-muted-foreground/20 pl-4'
)}
>
{@render children()}
</div>
</div>
</CollapsibleRegion>
</Collapsible.Content>
{/if}
</Collapsible.Root>
tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte
+8 -10
View File
@@ -32,7 +32,7 @@
UI_DATA_ATTRS
} from '$lib/constants';
import { BooleanString, ColorMode, UrlProtocol } from '$lib/enums';
import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
import { createStickToBottom } from '$lib/hooks/use-stick-to-bottom.svelte';
import { settingsStore } from '$lib/stores';
import type { DatabaseMessageExtra } from '$lib/types/database';
import {
@@ -123,8 +123,8 @@
let streamingCodeScrollContainer = $state<HTMLDivElement>();
// Auto-scroll controller for streaming code block content
const streamingAutoScroll = createAutoScrollController();
// Follows the code that streams into the fixed-height block below.
const streamingScroll = createStickToBottom();
let pendingMarkdown: string | null = null;
let isProcessing = false;
@@ -800,18 +800,16 @@
}
});
// Auto-scroll for streaming code block
// Follow the streaming code block while the reader stays at its bottom; the
// resize observer does the pinning, so no polling for a stable height.
$effect(() => {
streamingAutoScroll.setContainer(streamingCodeScrollContainer);
});
streamingScroll.attach(streamingCodeScrollContainer);
$effect(() => {
streamingAutoScroll.updateInterval(incompleteCodeBlock !== null);
return () => streamingScroll.detach();
});
onDestroy(() => {
cleanupEventListeners();
streamingAutoScroll.destroy();
});
</script>
@@ -905,7 +903,7 @@
<div
bind:this={streamingCodeScrollContainer}
class="streaming-code-scroll-container"
onscroll={() => streamingAutoScroll.handleScroll()}
onscroll={() => streamingScroll.handleScroll()}
>
<pre class="streaming-code-pre"><code
class="hljs language-{incompleteCodeBlock.language || 'text'}"
tools/ui/src/lib/components/app/dialogs/DialogConversationDelete.svelte
+24
View File
@@ -0,0 +1,24 @@
<script lang="ts">
import { Trash2 } from '@lucide/svelte';
import { DialogConfirmation } from '$lib/components/app';
interface Props {
open: boolean;
onConfirm: () => void;
onCancel: () => void;
}
let { onCancel, onConfirm, open = $bindable() }: Props = $props();
</script>
<DialogConfirmation
bind:open
cancelText="Cancel"
confirmText="Delete"
description="Are you sure you want to delete this conversation? This action cannot be undone and will permanently remove all messages in this conversation."
icon={Trash2}
{onCancel}
{onConfirm}
title="Delete Conversation"
variant="destructive"
/>
Loaded 100 of 151 files, more files were not shown because too many files have changed in this diff. Show more