upgrade tantivy (#6658)

This commit is contained in:
trinity-1686a authored and GitHub committed 2026-08-05 10:03:54 +02:00
1 parent c05e56df39
commit a8479d700b
5 files changed
+181 -38

No files matched your search

+36 -35
View File
@@ -199,7 +199,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -210,7 +210,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -1553,7 +1553,7 @@ dependencies = [
"bitflags 2.13.0",
"cexpr",
"clang-sys",
"itertools 0.13.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"regex",
@@ -1700,7 +1700,7 @@ version = "3.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
dependencies = [
"darling 0.23.0",
"darling 0.20.11",
"ident_case",
"prettyplease",
"proc-macro2",
@@ -2189,7 +2189,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]
@@ -2702,6 +2702,7 @@ dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.118",
]
@@ -4024,7 +4025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -5425,7 +5426,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -6433,7 +6434,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -6614,7 +6615,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"base64 0.21.7",
"chrono",
"getrandom 0.2.17",
"http 1.4.2",
@@ -7041,7 +7042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]
@@ -7077,7 +7078,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "ownedbytes"
version = "0.9.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"stable_deref_trait",
]
@@ -8051,7 +8052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"itertools 0.10.5",
"log",
"multimap",
"once_cell",
@@ -8071,7 +8072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"itertools 0.10.5",
"log",
"multimap",
"petgraph 0.8.3",
@@ -8092,7 +8093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.118",
@@ -8105,7 +8106,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.118",
@@ -9542,7 +9543,7 @@ dependencies = [
"once_cell",
"socket2 0.6.4",
"tracing",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -10308,7 +10309,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -10321,7 +10322,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -10401,7 +10402,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -10666,7 +10667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -11224,7 +11225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -11496,7 +11497,7 @@ dependencies = [
"cfg-if",
"libc",
"psm",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -11746,8 +11747,8 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tantivy"
version = "0.26.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
version = "0.27.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"aho-corasick",
"arc-swap",
@@ -11786,7 +11787,7 @@ dependencies = [
"tantivy-columnar",
"tantivy-common",
"tantivy-fst",
"tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03)",
"tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=86641f7)",
"tantivy-sstable",
"tantivy-stacker",
"tantivy-tokenizer-api",
@@ -11803,7 +11804,7 @@ dependencies = [
[[package]]
name = "tantivy-bitpacker"
version = "0.10.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"bitpacking",
]
@@ -11811,7 +11812,7 @@ dependencies = [
[[package]]
name = "tantivy-columnar"
version = "0.7.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"downcast-rs",
"fastdivide",
@@ -11826,7 +11827,7 @@ dependencies = [
[[package]]
name = "tantivy-common"
version = "0.11.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"async-trait",
"byteorder",
@@ -11862,7 +11863,7 @@ dependencies = [
[[package]]
name = "tantivy-query-grammar"
version = "0.26.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"fnv",
"nom 7.1.3",
@@ -11874,7 +11875,7 @@ dependencies = [
[[package]]
name = "tantivy-sstable"
version = "0.7.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"futures-util",
"itertools 0.14.0",
@@ -11887,7 +11888,7 @@ dependencies = [
[[package]]
name = "tantivy-stacker"
version = "0.7.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"murmurhash32",
"tantivy-common",
@@ -11896,7 +11897,7 @@ dependencies = [
[[package]]
name = "tantivy-tokenizer-api"
version = "0.7.0"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=70f0b03#70f0b039f5ea92ec6c38be4904f774b9d3444e65"
source = "git+https://github.com/quickwit-oss/tantivy/?rev=86641f7#86641f72df7f79867486aa4645b66767a80cc4e8"
dependencies = [
"serde",
]
@@ -11911,7 +11912,7 @@ dependencies = [
"getrandom 0.4.3",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -11920,7 +11921,7 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -13376,7 +13377,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]
+1 -1
View File
@@ -426,7 +426,7 @@ quickwit-storage = { path = "quickwit-storage" }
quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" }
quickwit-transport = { path = "quickwit-transport" }
tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "70f0b03", default-features = false, features = [
tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "86641f7", default-features = false, features = [
"lz4-compression",
"mmap",
"quickwit",
@@ -529,7 +529,7 @@ impl From<QuickwitTextOptions> for TextOptions {
}
match &quickwit_text_options.fast {
FastFieldOptions::EnabledWithNormalizer { normalizer } => {
text_options = text_options.set_fast(Some(normalizer.get_name()));
text_options = text_options.set_fast(normalizer.get_name());
}
FastFieldOptions::Disabled => {}
}
@@ -647,7 +647,7 @@ impl From<QuickwitJsonOptions> for JsonObjectOptions {
}
match &quickwit_json_options.fast {
FastFieldOptions::EnabledWithNormalizer { normalizer } => {
json_options = json_options.set_fast(Some(normalizer.get_name()));
json_options = json_options.set_fast(normalizer.get_name());
}
FastFieldOptions::Disabled => {}
}
@@ -20,6 +20,7 @@ use tantivy::aggregation::agg_result::{
BucketEntries as TantivyBucketEntries, BucketEntry as TantivyBucketEntry,
BucketResult as TantivyBucketResult, CompositeBucketEntry as TantivyCompositeBucketEntry,
CompositeKey as TantivyCompositeKey, MetricResult as TantivyMetricResult,
MultiTermsBucketEntry as TantivyMultiTermsBucketEntry,
RangeBucketEntry as TantivyRangeBucketEntry,
};
use tantivy::aggregation::bucket::AfterKey as TantivyAfterKey;
@@ -180,6 +181,17 @@ pub enum BucketResult {
/// serialization needed for correct pagination round-tripping.
after_key: FxHashMap<String, TantivyAfterKey>,
},
/// This is the multi_terms result
MultiTerms {
/// The buckets, one per unique combination of field values.
///
/// See `MultiTermsAggregation`
buckets: Vec<MultiTermsBucketEntry>,
/// The number of documents that didn’t make it into to TOP N due to shard_size or size
sum_other_doc_count: u64,
/// The upper bound error for the doc count of each term combination.
doc_count_error_upper_bound: Option<u64>,
},
}
impl From<TantivyBucketResult> for BucketResult {
@@ -207,6 +219,15 @@ impl From<TantivyBucketResult> for BucketResult {
buckets: buckets.into_iter().map(Into::into).collect(),
after_key,
},
TantivyBucketResult::MultiTerms {
buckets,
sum_other_doc_count,
doc_count_error_upper_bound,
} => BucketResult::MultiTerms {
buckets: buckets.into_iter().map(Into::into).collect(),
sum_other_doc_count,
doc_count_error_upper_bound,
},
}
}
}
@@ -233,6 +254,15 @@ impl From<BucketResult> for TantivyBucketResult {
buckets: buckets.into_iter().map(Into::into).collect(),
after_key,
},
BucketResult::MultiTerms {
buckets,
sum_other_doc_count,
doc_count_error_upper_bound,
} => TantivyBucketResult::MultiTerms {
buckets: buckets.into_iter().map(Into::into).collect(),
sum_other_doc_count,
doc_count_error_upper_bound,
},
}
}
}
@@ -358,6 +388,40 @@ impl From<BucketEntry> for TantivyBucketEntry {
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct MultiTermsBucketEntry {
/// Pipe-joined string representation of all key elements, e.g. `"rock|Product A"`.
pub key_as_string: String,
/// The composite key: one [`Key`] per field in declaration order.
pub key: Vec<Key>,
/// Number of documents in the bucket.
pub doc_count: u64,
/// Sub-aggregations in this bucket.
pub sub_aggregation: AggregationResults,
}
impl From<TantivyMultiTermsBucketEntry> for MultiTermsBucketEntry {
fn from(value: TantivyMultiTermsBucketEntry) -> MultiTermsBucketEntry {
MultiTermsBucketEntry {
key_as_string: value.key_as_string,
key: value.key.into_iter().map(Into::into).collect(),
doc_count: value.doc_count,
sub_aggregation: value.sub_aggregation.into(),
}
}
}
impl From<MultiTermsBucketEntry> for TantivyMultiTermsBucketEntry {
fn from(value: MultiTermsBucketEntry) -> TantivyMultiTermsBucketEntry {
TantivyMultiTermsBucketEntry {
key_as_string: value.key_as_string,
key: value.key.into_iter().map(Into::into).collect(),
doc_count: value.doc_count,
sub_aggregation: value.sub_aggregation.into(),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum Key {
/// String key
@@ -0,0 +1,78 @@
# Test multi_terms aggregation, ordered by key for a deterministic bucket order.
method: [GET]
engines:
- quickwit
endpoint: _elastic/aggregations/_search
json:
query: { match_all: {} }
aggs:
host_response:
multi_terms:
terms:
- field: "host"
- field: "response"
order: { _key: asc }
expected:
aggregations:
host_response:
buckets:
- key: ["192.168.0.1", 30]
key_as_string: "192.168.0.1|30"
doc_count: 1
- key: ["192.168.0.1", 100]
key_as_string: "192.168.0.1|100"
doc_count: 1
- key: ["192.168.0.10", 20]
key_as_string: "192.168.0.10|20"
doc_count: 1
- key: ["192.168.0.10", 30]
key_as_string: "192.168.0.10|30"
doc_count: 1
- key: ["192.168.0.10", 100]
key_as_string: "192.168.0.10|100"
doc_count: 1
- key: ["192.168.0.11", 120]
key_as_string: "192.168.0.11|120"
doc_count: 1
sum_other_doc_count: 0
---
# Test multi_terms aggregation with a multi-valued field and a sub-aggregation.
method: [GET]
engines:
- quickwit
endpoint: _elastic/aggregations/_search
json:
query: { match_all: {} }
aggs:
tags_response:
multi_terms:
terms:
- field: "tags"
- field: "response"
order: { _key: asc }
aggs:
avg_response:
avg:
field: "response"
expected:
aggregations:
tags_response:
buckets:
# Horst has tags [nice, cool] but no response, so he contributes no bucket.
- key: ["cool", 30]
key_as_string: "cool|30"
doc_count: 1
avg_response: { value: 30.0 }
- key: ["nice", 30]
key_as_string: "nice|30"
doc_count: 1
avg_response: { value: 30.0 }
- key: ["nice", 100]
key_as_string: "nice|100"
doc_count: 2
avg_response: { value: 100.0 }
- key: ["nice", 120]
key_as_string: "nice|120"
doc_count: 1
avg_response: { value: 120.0 }
sum_other_doc_count: 0