mirror of
https://github.com/qdrant/qdrant.git
synced 2026-10-28 16:44:42 +08:00
Kumar Shivendu
2e47b186fe
* feat(text-index): rank exact identifiers first with the code tokenizer
Ranking queries (BM25 text query, BM25 inference) now get joined
identifiers too, not only documents: a query for `getUserName`,
`get_user_name` or `getusername` also matches `getusername`, so a point
with that exact identifier outranks one with the same words scattered.
Filter and phrase queries are unchanged, since they need every token.
Documents now also join `snake_case` identifiers (`get_user_name` ->
`getusername`), so a camelCase or lowercase query finds them too.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* refactor(text-index): route ranking queries through TokenizerTextKind
Add `TokenizerTextKind::RankingQuery` and dispatch it in `Tokenizer::tokenize`,
instead of `tokenize_ranking_query` special-casing the code tokenizer.
Rename the code tokenizer methods by what they emit: `tokenize_with_joined`
(subwords + joined identifiers) and `tokenize_subwords`.
Strip `ARRAY_BOUNDARY_SENTINEL` inside `tokenize` for every query kind, so
direct `tokenize(Query | Phrase)` callers no longer see it either.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: generall <andrey@vasnetsov.com>
(cherry picked from commit 6708deb95b)
80 lines
2.8 KiB
Python
80 lines
2.8 KiB
Python
import pytest
|
|
|
|
from .helpers.collection_setup import drop_collection
|
|
from .helpers.helpers import request_with_validation
|
|
|
|
FIELD = "body"
|
|
|
|
# The same words, as an identifier or in a comment.
|
|
DOCUMENTS = {
|
|
1: "def getUserName(self): return self.name",
|
|
2: "# get the name of a user",
|
|
3: "def get_user_name(self): return self.name",
|
|
}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def setup(collection_name):
|
|
drop_collection(collection_name=collection_name)
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
body={"sparse_vectors": {}},
|
|
)
|
|
assert response.ok, response.text
|
|
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/index',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
query_params={'wait': 'true'},
|
|
body={"field_name": FIELD,
|
|
"field_schema": {"type": "text", "tokenizer": "code", "scoring": True}},
|
|
)
|
|
assert response.ok, response.text
|
|
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
query_params={'wait': 'true'},
|
|
body={"points": [{"id": i, "vector": {}, "payload": {FIELD: text}}
|
|
for i, text in DOCUMENTS.items()]},
|
|
)
|
|
assert response.ok, response.text
|
|
yield
|
|
drop_collection(collection_name=collection_name)
|
|
|
|
|
|
def rank(collection_name, query):
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/query',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={"query": {"text": query}, "using": FIELD, "limit": 10},
|
|
)
|
|
assert response.ok, response.text
|
|
return [(point["id"], point["score"]) for point in response.json()["result"]["points"]]
|
|
|
|
|
|
def test_exact_identifier_ranks_first(collection_name):
|
|
# Only the identifiers have the joined token `getusername`
|
|
for query in ["getUserName", "get_user_name", "getusername"]:
|
|
ranked = rank(collection_name, query)
|
|
assert {point_id for point_id, _ in ranked[:2]} == {1, 3}, (query, ranked)
|
|
assert all(score < ranked[1][1] for _, score in ranked[2:]), (query, ranked)
|
|
|
|
|
|
def test_filter_needs_subwords_only(collection_name):
|
|
# The comment has the subwords but not the joined token
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/scroll',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={"filter": {"must": [{"key": FIELD, "match": {"text": "getUserName"}}]},
|
|
"limit": 10},
|
|
)
|
|
assert response.ok, response.text
|
|
assert {point["id"] for point in response.json()["result"]["points"]} == set(DOCUMENTS)
|