From a8479d700bd14899a78ff650940b60b1b8f416e2 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 5 Aug 2026 10:03:54 +0200 Subject: [PATCH] upgrade tantivy (#6658) --- quickwit/Cargo.lock | 71 ++++++++--------- quickwit/Cargo.toml | 2 +- .../src/doc_mapper/field_mapping_entry.rs | 4 +- quickwit/quickwit-query/src/aggregations.rs | 64 +++++++++++++++ .../aggregations/0003-multi-terms.yaml | 78 +++++++++++++++++++ 5 files changed, 181 insertions(+), 38 deletions(-) create mode 100644 quickwit/rest-api-tests/scenarii/aggregations/0003-multi-terms.yaml diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 448e8b3db..b5454049f 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -199,7 +199,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -210,7 +210,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1553,7 +1553,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -1700,7 +1700,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.20.11", "ident_case", "prettyplease", "proc-macro2", @@ -2189,7 +2189,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2702,6 +2702,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", + "strsim", "syn 2.0.118", ] @@ -4024,7 +4025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5425,7 +5426,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6433,7 +6434,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6614,7 +6615,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.4.2", @@ -7041,7 +7042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -7077,7 +7078,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "stable_deref_trait", ] @@ -8051,7 +8052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -8071,7 +8072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph 0.8.3", @@ -8092,7 +8093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -8105,7 +8106,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -9542,7 +9543,7 @@ dependencies = [ "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10308,7 +10309,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -10321,7 +10322,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10401,7 +10402,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10666,7 +10667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11224,7 +11225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -11496,7 +11497,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -11746,8 +11747,8 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.26.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +version = "0.27.0" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "aho-corasick", "arc-swap", @@ -11786,7 +11787,7 @@ dependencies = [ "tantivy-columnar", "tantivy-common", "tantivy-fst", - "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03)", + "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=86641f7)", "tantivy-sstable", "tantivy-stacker", "tantivy-tokenizer-api", @@ -11803,7 +11804,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.10.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "bitpacking", ] @@ -11811,7 +11812,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "downcast-rs", "fastdivide", @@ -11826,7 +11827,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.11.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "async-trait", "byteorder", @@ -11862,7 +11863,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.26.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "fnv", "nom 7.1.3", @@ -11874,7 +11875,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "futures-util", "itertools 0.14.0", @@ -11887,7 +11888,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "murmurhash32", "tantivy-common", @@ -11896,7 +11897,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8" dependencies = [ "serde", ] @@ -11911,7 +11912,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11920,7 +11921,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -13376,7 +13377,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index c91169d38..f5ace1bef 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -426,7 +426,7 @@ quickwit-storage = { path = "quickwit-storage" } quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" } quickwit-transport = { path = "quickwit-transport" } -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "70f0b03", default-features = false, features = [ +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "86641f7", default-features = false, features = [ "lz4-compression", "mmap", "quickwit", diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs index f0043d281..2f3e75534 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/field_mapping_entry.rs @@ -529,7 +529,7 @@ impl From for TextOptions { } match &quickwit_text_options.fast { FastFieldOptions::EnabledWithNormalizer { normalizer } => { - text_options = text_options.set_fast(Some(normalizer.get_name())); + text_options = text_options.set_fast(normalizer.get_name()); } FastFieldOptions::Disabled => {} } @@ -647,7 +647,7 @@ impl From for JsonObjectOptions { } match &quickwit_json_options.fast { FastFieldOptions::EnabledWithNormalizer { normalizer } => { - json_options = json_options.set_fast(Some(normalizer.get_name())); + json_options = json_options.set_fast(normalizer.get_name()); } FastFieldOptions::Disabled => {} } diff --git a/quickwit/quickwit-query/src/aggregations.rs b/quickwit/quickwit-query/src/aggregations.rs index 1311a95c2..94c697286 100644 --- a/quickwit/quickwit-query/src/aggregations.rs +++ b/quickwit/quickwit-query/src/aggregations.rs @@ -20,6 +20,7 @@ use tantivy::aggregation::agg_result::{ BucketEntries as TantivyBucketEntries, BucketEntry as TantivyBucketEntry, BucketResult as TantivyBucketResult, CompositeBucketEntry as TantivyCompositeBucketEntry, CompositeKey as TantivyCompositeKey, MetricResult as TantivyMetricResult, + MultiTermsBucketEntry as TantivyMultiTermsBucketEntry, RangeBucketEntry as TantivyRangeBucketEntry, }; use tantivy::aggregation::bucket::AfterKey as TantivyAfterKey; @@ -180,6 +181,17 @@ pub enum BucketResult { /// serialization needed for correct pagination round-tripping. after_key: FxHashMap, }, + /// This is the multi_terms result + MultiTerms { + /// The buckets, one per unique combination of field values. + /// + /// See `MultiTermsAggregation` + buckets: Vec, + /// The number of documents that didn’t make it into to TOP N due to shard_size or size + sum_other_doc_count: u64, + /// The upper bound error for the doc count of each term combination. + doc_count_error_upper_bound: Option, + }, } impl From for BucketResult { @@ -207,6 +219,15 @@ impl From for BucketResult { buckets: buckets.into_iter().map(Into::into).collect(), after_key, }, + TantivyBucketResult::MultiTerms { + buckets, + sum_other_doc_count, + doc_count_error_upper_bound, + } => BucketResult::MultiTerms { + buckets: buckets.into_iter().map(Into::into).collect(), + sum_other_doc_count, + doc_count_error_upper_bound, + }, } } } @@ -233,6 +254,15 @@ impl From for TantivyBucketResult { buckets: buckets.into_iter().map(Into::into).collect(), after_key, }, + BucketResult::MultiTerms { + buckets, + sum_other_doc_count, + doc_count_error_upper_bound, + } => TantivyBucketResult::MultiTerms { + buckets: buckets.into_iter().map(Into::into).collect(), + sum_other_doc_count, + doc_count_error_upper_bound, + }, } } } @@ -358,6 +388,40 @@ impl From for TantivyBucketEntry { } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MultiTermsBucketEntry { + /// Pipe-joined string representation of all key elements, e.g. `"rock|Product A"`. + pub key_as_string: String, + /// The composite key: one [`Key`] per field in declaration order. + pub key: Vec, + /// Number of documents in the bucket. + pub doc_count: u64, + /// Sub-aggregations in this bucket. + pub sub_aggregation: AggregationResults, +} + +impl From for MultiTermsBucketEntry { + fn from(value: TantivyMultiTermsBucketEntry) -> MultiTermsBucketEntry { + MultiTermsBucketEntry { + key_as_string: value.key_as_string, + key: value.key.into_iter().map(Into::into).collect(), + doc_count: value.doc_count, + sub_aggregation: value.sub_aggregation.into(), + } + } +} + +impl From for TantivyMultiTermsBucketEntry { + fn from(value: MultiTermsBucketEntry) -> TantivyMultiTermsBucketEntry { + TantivyMultiTermsBucketEntry { + key_as_string: value.key_as_string, + key: value.key.into_iter().map(Into::into).collect(), + doc_count: value.doc_count, + sub_aggregation: value.sub_aggregation.into(), + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub enum Key { /// String key diff --git a/quickwit/rest-api-tests/scenarii/aggregations/0003-multi-terms.yaml b/quickwit/rest-api-tests/scenarii/aggregations/0003-multi-terms.yaml new file mode 100644 index 000000000..8dafd0698 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/aggregations/0003-multi-terms.yaml @@ -0,0 +1,78 @@ +# Test multi_terms aggregation, ordered by key for a deterministic bucket order. +method: [GET] +engines: + - quickwit +endpoint: _elastic/aggregations/_search +json: + query: { match_all: {} } + aggs: + host_response: + multi_terms: + terms: + - field: "host" + - field: "response" + order: { _key: asc } +expected: + aggregations: + host_response: + buckets: + - key: ["192.168.0.1", 30] + key_as_string: "192.168.0.1|30" + doc_count: 1 + - key: ["192.168.0.1", 100] + key_as_string: "192.168.0.1|100" + doc_count: 1 + - key: ["192.168.0.10", 20] + key_as_string: "192.168.0.10|20" + doc_count: 1 + - key: ["192.168.0.10", 30] + key_as_string: "192.168.0.10|30" + doc_count: 1 + - key: ["192.168.0.10", 100] + key_as_string: "192.168.0.10|100" + doc_count: 1 + - key: ["192.168.0.11", 120] + key_as_string: "192.168.0.11|120" + doc_count: 1 + sum_other_doc_count: 0 +--- +# Test multi_terms aggregation with a multi-valued field and a sub-aggregation. +method: [GET] +engines: + - quickwit +endpoint: _elastic/aggregations/_search +json: + query: { match_all: {} } + aggs: + tags_response: + multi_terms: + terms: + - field: "tags" + - field: "response" + order: { _key: asc } + aggs: + avg_response: + avg: + field: "response" +expected: + aggregations: + tags_response: + buckets: + # Horst has tags [nice, cool] but no response, so he contributes no bucket. + - key: ["cool", 30] + key_as_string: "cool|30" + doc_count: 1 + avg_response: { value: 30.0 } + - key: ["nice", 30] + key_as_string: "nice|30" + doc_count: 1 + avg_response: { value: 30.0 } + - key: ["nice", 100] + key_as_string: "nice|100" + doc_count: 2 + avg_response: { value: 100.0 } + - key: ["nice", 120] + key_as_string: "nice|120" + doc_count: 1 + avg_response: { value: 120.0 } + sum_other_doc_count: 0