Files
ClickHouse/.gitmodules
Konstantin Vedernikov 0ae7ef5ef7 GDeflate compression codec on the CPU
`CODEC(GDeflate[(level)])`, behind `enable_gdeflate_codec`, levels 1..12,
default 6, method byte 0xa1. GDeflate is a variant of Deflate whose bit stream
is split into 32 interleaved sub-streams, the format that nvcomp and
DirectStorage expand on a GPU, many chunks in parallel.

A block is compressed in independent chunks of 64 KiB, each a GDeflate page,
from a multiple of 4 bytes, after the number of chunks and a descriptor of
each. Of a column of integers, `Int*`, `UInt*`, `Decimal32`, `Decimal64`,
`Date*` or `DateTime*`, a chunk is also bit-packed before it is compressed:
each integer as its difference from the least of the chunk, in as many bits as
the largest difference takes, up to 56. The smaller of the two forms is kept,
and the block says which. The format is that of the GPU codec of
https://github.com/ClickHouse/ClickHouse/pull/119502, so that a GPU can expand
the chunks of a column written by this codec.

The codec uses the GDeflate pages of NVIDIA's fork of libdeflate
(`contrib/gdeflate`), whose compressed pages nvcomp expands on the GPU, and
whose expansion reads what nvcomp compresses, as checked on the columns of
TPC-H `lineitem` at every level. The fork exports the same symbols as
libdeflate, so `contrib/gdeflate-cmake` builds only its GDeflate parts and
renames them.

`contrib/gdeflate` points at the upstream repository; it must be replaced with
a fork under github.com/ClickHouse before merging.
2026-10-22 14:35:39 +00:00

483 lines
16 KiB
Plaintext

# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
# more confusing than useful.
[submodule "contrib/jwt-cpp"]
path = contrib/jwt-cpp
url = https://github.com/Thalhammer/jwt-cpp
[submodule "contrib/zstd"]
path = contrib/zstd
url = https://github.com/ClickHouse/zstd
[submodule "contrib/lz4"]
path = contrib/lz4
url = https://github.com/lz4/lz4
[submodule "contrib/librdkafka"]
path = contrib/librdkafka
url = https://github.com/ClickHouse/librdkafka
[submodule "contrib/cctz"]
path = contrib/cctz
url = https://github.com/ClickHouse/cctz
[submodule "contrib/zlib-ng"]
path = contrib/zlib-ng
url = https://github.com/ClickHouse/zlib-ng
[submodule "contrib/googletest"]
path = contrib/googletest
url = https://github.com/google/googletest
[submodule "contrib/capnproto"]
path = contrib/capnproto
url = https://github.com/ClickHouse/capnproto
[submodule "contrib/re2"]
path = contrib/re2
url = https://github.com/ClickHouse/re2
[submodule "contrib/mariadb-connector-c"]
path = contrib/mariadb-connector-c
url = https://github.com/ClickHouse/mariadb-connector-c
[submodule "contrib/jemalloc"]
path = contrib/jemalloc
url = https://github.com/ClickHouse/jemalloc
[submodule "contrib/google-protobuf"]
path = contrib/google-protobuf
url = https://github.com/protocolbuffers/protobuf.git
[submodule "contrib/boost"]
path = contrib/boost
url = https://github.com/ClickHouse/boost
[submodule "contrib/arrow"]
path = contrib/arrow
url = https://github.com/ClickHouse/arrow
[submodule "contrib/thrift"]
path = contrib/thrift
url = https://github.com/ClickHouse/thrift
[submodule "contrib/libhdfs3"]
path = contrib/libhdfs3
url = https://github.com/ClickHouse/libhdfs3
[submodule "contrib/libxml2"]
path = contrib/libxml2
url = https://github.com/GNOME/libxml2
[submodule "contrib/libgsasl"]
path = contrib/libgsasl
url = https://github.com/ClickHouse/libgsasl
[submodule "contrib/snappy"]
path = contrib/snappy
url = https://github.com/ClickHouse/snappy
[submodule "contrib/cppkafka"]
path = contrib/cppkafka
url = https://github.com/ClickHouse/cppkafka
[submodule "contrib/brotli"]
path = contrib/brotli
url = https://github.com/ClickHouse/brotli
[submodule "contrib/h3"]
path = contrib/h3
url = https://github.com/ClickHouse/h3
[submodule "contrib/simdjson"]
path = contrib/simdjson
url = https://github.com/ClickHouse/simdjson
[submodule "contrib/rapidjson"]
path = contrib/rapidjson
url = https://github.com/ClickHouse/rapidjson
[submodule "contrib/fastops"]
path = contrib/fastops
url = https://github.com/ClickHouse/fastops
[submodule "contrib/orc"]
path = contrib/orc
url = https://github.com/ClickHouse/orc
[submodule "contrib/sparsehash-c11"]
path = contrib/sparsehash-c11
url = https://github.com/sparsehash/sparsehash-c11
[submodule "contrib/grpc"]
path = contrib/grpc
url = https://github.com/ClickHouse/grpc
[submodule "contrib/icudata"]
path = contrib/icudata
url = https://github.com/ClickHouse/icudata
[submodule "contrib/icu"]
path = contrib/icu
url = https://github.com/ClickHouse/icu
[submodule "contrib/flatbuffers"]
path = contrib/flatbuffers
url = https://github.com/google/flatbuffers
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/ClickHouse/replxx
[submodule "contrib/avro"]
path = contrib/avro
url = https://github.com/ClickHouse/avro
ignore = untracked
[submodule "contrib/msgpack-c"]
path = contrib/msgpack-c
url = https://github.com/ClickHouse/msgpack-c
[submodule "contrib/libcpuid"]
path = contrib/libcpuid
url = https://github.com/anrieff/libcpuid
[submodule "contrib/openldap"]
path = contrib/openldap
url = https://github.com/openldap/openldap
[submodule "contrib/AMQP-CPP"]
path = contrib/AMQP-CPP
url = https://github.com/ClickHouse/AMQP-CPP
[submodule "contrib/libuv"]
path = contrib/libuv
url = https://github.com/ClickHouse/libuv
[submodule "contrib/fmtlib"]
path = contrib/fmtlib
url = https://github.com/ClickHouse/fmt
[submodule "contrib/krb5"]
path = contrib/krb5
url = https://github.com/ClickHouse/krb5
[submodule "contrib/cyrus-sasl"]
path = contrib/cyrus-sasl
url = https://github.com/ClickHouse/cyrus-sasl
[submodule "contrib/croaring"]
path = contrib/croaring
url = https://github.com/RoaringBitmap/CRoaring
[submodule "contrib/miniselect"]
path = contrib/miniselect
url = https://github.com/danlark1/miniselect
[submodule "contrib/rocksdb"]
path = contrib/rocksdb
url = https://github.com/ClickHouse/rocksdb
[submodule "contrib/xz"]
path = contrib/xz
url = https://github.com/tukaani-project/xz
[submodule "contrib/abseil-cpp"]
path = contrib/abseil-cpp
url = https://github.com/abseil/abseil-cpp.git
[submodule "contrib/fast_float"]
path = contrib/fast_float
url = https://github.com/ClickHouse/fast_float
[submodule "contrib/NuRaft"]
path = contrib/NuRaft
url = https://github.com/ClickHouse/NuRaft
[submodule "contrib/datasketches-cpp"]
path = contrib/datasketches-cpp
url = https://github.com/ClickHouse/datasketches-cpp
[submodule "contrib/yaml-cpp"]
path = contrib/yaml-cpp
url = https://github.com/ClickHouse/yaml-cpp
[submodule "contrib/cld2"]
path = contrib/cld2
url = https://github.com/ClickHouse/cld2
[submodule "contrib/libstemmer_c"]
path = contrib/libstemmer_c
url = https://github.com/ClickHouse/libstemmer_c
[submodule "contrib/wordnet-blast"]
path = contrib/wordnet-blast
url = https://github.com/ClickHouse/wordnet-blast
[submodule "contrib/lemmagen-c"]
path = contrib/lemmagen-c
url = https://github.com/ClickHouse/lemmagen-c
[submodule "contrib/libpqxx"]
path = contrib/libpqxx
url = https://github.com/ClickHouse/libpqxx
[submodule "contrib/sqlite-amalgamation"]
path = contrib/sqlite-amalgamation
url = https://github.com/ClickHouse/sqlite-amalgamation
[submodule "contrib/s2geometry"]
path = contrib/s2geometry
url = https://github.com/ClickHouse/s2geometry
[submodule "contrib/bzip2"]
path = contrib/bzip2
url = https://github.com/ClickHouse/bzip2
[submodule "contrib/magic_enum"]
path = contrib/magic_enum
url = https://github.com/Neargye/magic_enum
[submodule "contrib/libprotobuf-mutator"]
path = contrib/libprotobuf-mutator
url = https://github.com/google/libprotobuf-mutator
[submodule "contrib/sysroot"]
path = contrib/sysroot
url = https://github.com/ClickHouse/sysroot
[submodule "contrib/hive-metastore"]
path = contrib/hive-metastore
url = https://github.com/ClickHouse/hive-metastore
[submodule "contrib/azure"]
path = contrib/azure
url = https://github.com/ClickHouse/azure-sdk-for-cpp
[submodule "contrib/minizip-ng"]
path = contrib/minizip-ng
url = https://github.com/zlib-ng/minizip-ng
[submodule "contrib/wyhash"]
path = contrib/wyhash
url = https://github.com/wangyi-fudan/wyhash
[submodule "contrib/nats-io"]
path = contrib/nats-io
url = https://github.com/ClickHouse/nats.c
[submodule "contrib/vectorscan"]
path = contrib/vectorscan
url = https://github.com/ClickHouse/vectorscan
[submodule "contrib/llvm-project"]
path = contrib/llvm-project
url = https://github.com/ClickHouse/llvm-project
[submodule "contrib/corrosion"]
path = contrib/corrosion
url = https://github.com/corrosion-rs/corrosion
[submodule "contrib/libssh"]
path = contrib/libssh
url = https://github.com/ClickHouse/libssh.git
[submodule "contrib/morton-nd"]
path = contrib/morton-nd
url = https://github.com/morton-nd/morton-nd
[submodule "contrib/xxHash"]
path = contrib/xxHash
url = https://github.com/Cyan4973/xxHash
[submodule "contrib/crc32-s390x"]
path = contrib/crc32-s390x
url = https://github.com/linux-on-ibm-z/crc32-s390x
[submodule "contrib/google-benchmark"]
path = contrib/google-benchmark
url = https://github.com/google/benchmark
[submodule "contrib/libdivide"]
path = contrib/libdivide
url = https://github.com/ridiculousfish/libdivide
[submodule "contrib/libbcrypt"]
path = contrib/libbcrypt
url = https://github.com/rg3/libbcrypt.git
[submodule "contrib/ulid-c"]
path = contrib/ulid-c
url = https://github.com/ClickHouse/ulid-c.git
[submodule "contrib/aws"]
path = contrib/aws
url = https://github.com/ClickHouse/aws-sdk-cpp
[submodule "contrib/aws-c-event-stream"]
path = contrib/aws-c-event-stream
url = https://github.com/awslabs/aws-c-event-stream
[submodule "contrib/aws-c-common"]
path = contrib/aws-c-common
url = https://github.com/awslabs/aws-c-common.git
[submodule "contrib/aws-checksums"]
path = contrib/aws-checksums
url = https://github.com/awslabs/aws-checksums
[submodule "contrib/aws-crt-cpp"]
path = contrib/aws-crt-cpp
url = https://github.com/ClickHouse/aws-crt-cpp
[submodule "contrib/aws-c-io"]
path = contrib/aws-c-io
url = https://github.com/ClickHouse/aws-c-io
[submodule "contrib/aws-c-mqtt"]
path = contrib/aws-c-mqtt
url = https://github.com/awslabs/aws-c-mqtt
[submodule "contrib/aws-c-auth"]
path = contrib/aws-c-auth
url = https://github.com/awslabs/aws-c-auth
[submodule "contrib/aws-c-cal"]
path = contrib/aws-c-cal
url = https://github.com/awslabs/aws-c-cal
[submodule "contrib/aws-c-sdkutils"]
path = contrib/aws-c-sdkutils
url = https://github.com/awslabs/aws-c-sdkutils
[submodule "contrib/aws-c-http"]
path = contrib/aws-c-http
url = https://github.com/awslabs/aws-c-http
[submodule "contrib/aws-c-s3"]
path = contrib/aws-c-s3
url = https://github.com/awslabs/aws-c-s3
[submodule "contrib/aws-c-compression"]
path = contrib/aws-c-compression
url = https://github.com/awslabs/aws-c-compression
[submodule "contrib/crc32-vpmsum"]
path = contrib/crc32-vpmsum
url = https://github.com/antonblanchard/crc32-vpmsum.git
[submodule "contrib/liburing"]
path = contrib/liburing
url = https://github.com/axboe/liburing
[submodule "contrib/libarchive"]
path = contrib/libarchive
url = https://github.com/libarchive/libarchive.git
ignore = dirty
[submodule "contrib/libfiu"]
path = contrib/libfiu
url = https://github.com/ClickHouse/libfiu.git
[submodule "contrib/isa-l"]
path = contrib/isa-l
url = https://github.com/ClickHouse/isa-l.git
[submodule "contrib/c-ares"]
path = contrib/c-ares
url = https://github.com/c-ares/c-ares.git
[submodule "contrib/usearch"]
path = contrib/usearch
url = https://github.com/ClickHouse/usearch.git
[submodule "contrib/SimSIMD"]
path = contrib/SimSIMD
url = https://github.com/ClickHouse/SimSIMD.git
[submodule "contrib/FP16"]
path = contrib/FP16
url = https://github.com/Maratyszcza/FP16.git
[submodule "contrib/pocketfft"]
path = contrib/pocketfft
url = https://github.com/mreineck/pocketfft.git
[submodule "contrib/sqids-cpp"]
path = contrib/sqids-cpp
url = https://github.com/sqids/sqids-cpp.git
[submodule "contrib/idna"]
path = contrib/idna
url = https://github.com/ClickHouse/idna
[submodule "contrib/google-cloud-cpp"]
path = contrib/google-cloud-cpp
url = https://github.com/ClickHouse/google-cloud-cpp.git
[submodule "contrib/rust_vendor"]
path = contrib/rust_vendor
url = https://github.com/ClickHouse/rust_vendor.git
[submodule "contrib/openssl"]
path = contrib/openssl
url = https://github.com/ClickHouse/openssl.git
[submodule "contrib/double-conversion"]
path = contrib/double-conversion
url = https://github.com/google/double-conversion
[submodule "contrib/mongo-cxx-driver"]
path = contrib/mongo-cxx-driver
url = https://github.com/ClickHouse/mongo-cxx-driver.git
[submodule "contrib/mongo-c-driver"]
path = contrib/mongo-c-driver
url = https://github.com/ClickHouse/mongo-c-driver
[submodule "contrib/sz3"]
path = contrib/sz3
url = https://github.com/ClickHouse/SZ3.git
[submodule "contrib/postgres"]
path = contrib/postgres
url = https://github.com/ClickHouse/postgres.git
[submodule "contrib/wasmtime"]
path = contrib/wasmtime
url = https://github.com/ClickHouse/wasmtime
[submodule "contrib/delta-kernel-rs"]
path = contrib/delta-kernel-rs
url = https://github.com/ClickHouse/delta-kernel-rs.git
[submodule "contrib/SHA3IUF"]
path = contrib/SHA3IUF
url = https://github.com/brainhub/SHA3IUF.git
[submodule "contrib/xgboost"]
path = contrib/xgboost
url = https://github.com/ClickHouse/xgboost
[submodule "contrib/chdig"]
path = contrib/chdig
url = https://github.com/azat/chdig.git
[submodule "contrib/ai-sdk-cpp"]
path = contrib/ai-sdk-cpp
url = https://github.com/ClickHouse/ai-sdk-cpp
[submodule "contrib/nlohmann-json"]
path = contrib/nlohmann-json
url = https://github.com/ClickHouse/json
[submodule "contrib/crc32c"]
path = contrib/crc32c
url = https://github.com/ClickHouse/crc32c
[submodule "contrib/StringZilla"]
path = contrib/StringZilla
url = https://github.com/ClickHouse/StringZilla.git
[submodule "contrib/simdcomp"]
path = contrib/simdcomp
url = https://github.com/fast-pack/simdcomp.git
[submodule "contrib/antlr4-cpp-runtime"]
path = contrib/antlr4-cpp-runtime
url = https://github.com/antlr/antlr4.git
[submodule "contrib/darts-clone"]
path = contrib/darts-clone
url = https://github.com/s-yata/darts-clone
[submodule "contrib/libcotp"]
path = contrib/libcotp
url = https://github.com/paolostivanin/libcotp.git
[submodule "contrib/clickstack"]
path = contrib/clickstack
url = https://github.com/ClickHouse/clickhouse-clickstack.git
[submodule "contrib/ipcrypt2"]
path = contrib/ipcrypt2
url = https://github.com/jedisct1/ipcrypt2.git
[submodule "contrib/heks"]
path = contrib/heks
url = https://github.com/ClickHouse/heks.git
[submodule "contrib/musl"]
path = contrib/musl
url = https://github.com/ClickHouse/musl
[submodule "contrib/zmij"]
path = contrib/zmij
url = https://github.com/ClickHouse/zmij.git
[submodule "contrib/sql-console"]
path = contrib/sql-console
url = https://github.com/ClickHouse/clickhouse-sql-console.git
[submodule "contrib/librseq"]
path = contrib/librseq
url = https://github.com/compudj/librseq.git
[submodule "contrib/silk"]
path = contrib/silk
url = https://github.com/ClickHouse/silk.git
[submodule "contrib/xsimd"]
path = contrib/xsimd
url = https://github.com/ClickHouse/xsimd
[submodule "contrib/simdutf"]
path = contrib/simdutf
url = https://github.com/ClickHouse/simdutf
[submodule "contrib/libdeflate"]
path = contrib/libdeflate
url = https://github.com/ClickHouse/libdeflate
[submodule "contrib/wagyu"]
path = contrib/wagyu
url = https://github.com/ClickHouse/wagyu
[submodule "contrib/mapbox-geometry"]
path = contrib/mapbox-geometry
url = https://github.com/ClickHouse/geometry.hpp
[submodule "contrib/pcodec"]
path = contrib/pcodec
url = https://github.com/ClickHouse/pcodec.git
[submodule "contrib/zxc"]
path = contrib/zxc
url = https://github.com/hellobertrand/zxc
[submodule "contrib/dmlc-core"]
path = contrib/dmlc-core
url = https://github.com/dmlc/dmlc-core
[submodule "contrib/libucontext"]
path = contrib/libucontext
url = https://github.com/kaniini/libucontext
[submodule "contrib/Jieba-CPP"]
path = contrib/Jieba-CPP
url = https://github.com/amosbird/Jieba-CPP
[submodule "contrib/MeCab"]
path = contrib/MeCab
url = https://github.com/taku910/mecab
[submodule "contrib/substrait"]
path = contrib/substrait
url = https://github.com/substrait-io/substrait.git
[submodule "contrib/optimized-routines"]
path = contrib/optimized-routines
url = https://github.com/ARM-software/optimized-routines
[submodule "contrib/cudf"]
path = contrib/cudf
url = https://github.com/rapidsai/cudf.git
[submodule "contrib/rmm"]
path = contrib/rmm
url = https://github.com/rapidsai/rmm.git
[submodule "contrib/kvikio"]
path = contrib/kvikio
url = https://github.com/rapidsai/kvikio.git
[submodule "contrib/cccl"]
path = contrib/cccl
url = https://github.com/NVIDIA/cccl.git
[submodule "contrib/cuCollections"]
path = contrib/cuCollections
url = https://github.com/NVIDIA/cuCollections.git
[submodule "contrib/cucascade"]
path = contrib/cucascade
url = https://github.com/NVIDIA/cuCascade.git
[submodule "contrib/nvtx"]
path = contrib/nvtx
url = https://github.com/NVIDIA/NVTX.git
[submodule "contrib/rapids-logger"]
path = contrib/rapids-logger
url = https://github.com/rapidsai/rapids-logger.git
[submodule "contrib/bs-thread-pool"]
path = contrib/bs-thread-pool
url = https://github.com/bshoshany/thread-pool.git
[submodule "contrib/dlpack"]
path = contrib/dlpack
url = https://github.com/dmlc/dlpack.git
[submodule "contrib/nanoarrow"]
path = contrib/nanoarrow
url = https://github.com/apache/arrow-nanoarrow.git
[submodule "contrib/librtcx"]
path = contrib/librtcx
url = https://github.com/rapidsai/librtcx.git
[submodule "contrib/spdlog"]
path = contrib/spdlog
url = https://github.com/gabime/spdlog.git
[submodule "contrib/parquet-testing"]
path = contrib/parquet-testing
url = https://github.com/apache/parquet-testing
[submodule "contrib/gdeflate"]
path = contrib/gdeflate
url = https://github.com/NVIDIA/libdeflate.git