Compute the lexer keyword hash at compile time (#29677)

## Summary

Changing the lexer's keyword set now takes one edit to the Rust list.
The compiler finds a collision-free multiplier and builds the 64-slot
table, so we can remove the Python generator and its lockfile.

A two-byte keyword like `if` packs to `0x6669_0000_0000_0000`. Only the
low two bytes of the multiplier affect its slot. We place keywords from
shortest to longest, choosing each new byte of the multiplier without
disturbing the shorter keywords. The lexer still selects a slot with one
multiply and shift, then compares the full word.

crates/ruff_python_parser/src/lexer.rs

@@ -1602,21 +1602,15 @@ fn ascii_identifier_prefix_len(bytes: &[u8]) -> usize {
 ///
 /// Multiplying a packed keyword by `KEYWORD_HASH_MULTIPLIER` and taking the high six
 /// bits gives each keyword a distinct slot in the 64-entry table. This is a perfect
-/// hash for the keywords only; other identifiers can collide and need a full-word
-/// comparison.
+/// hash for the keywords. Other identifiers can collide and need a full-word comparison.
 ///
 /// `phf::Map` uses `SipHash` and a displacement table to find each candidate's slot.
 /// With the same length and first-byte filters, `phf` 0.11.3 took 12–21% longer
 /// to lex the NumPy and Pydantic benchmark files on Apple Silicon.
 ///
-/// To add, remove, or rename a keyword, update `KEYWORDS` in the generator, then run:
-///
-/// ```text
-/// uv run scripts/generate_keyword_hash.py
-/// ```
-///
-/// Keywords must contain two to eight ASCII bytes; supporting other lengths
-/// requires changing the word representation and length filter.
+/// Edit `KEYWORDS` in the `keywords` module to change the keyword set. Rust finds
+/// the multiplier and builds the table at compile time. Keywords must contain
+/// two to eight ASCII bytes to fit the word representation and length filter.
 #[repr(align(64))]
 struct KeywordTable {
     words: [u64; 64],
@@ -1639,20 +1633,10 @@ impl KeywordTable {
         let mut words = [0; 64];
         let mut kinds = [TokenKind::Identifier; 64];
         while let Some((&(keyword, kind), rest)) = keywords.split_first() {
-            assert!(
-                keyword.len() >= 2 && keyword.len() <= 8 && keyword.is_ascii(),
-                "keywords must contain two to eight ASCII bytes"
-            );
+            let word = keywords::pack(keyword);
             starts[keyword.len() - 2] |= 1u64 << (keyword[0] & 63);
-            let mut bytes = [0; 8];
-            let (keyword_bytes, _) = bytes.split_at_mut(keyword.len());
-            keyword_bytes.copy_from_slice(keyword);
-            let word = u64::from_le_bytes(bytes) << ((8 - keyword.len()) * 8);
-            let slot = (word.wrapping_mul(KEYWORD_HASH_MULTIPLIER) >> 58) as usize;
-            assert!(
-                words[slot] == 0,
-                "keyword hash collision: run `uv run scripts/generate_keyword_hash.py`"
-            );
+            let slot = keywords::slot(word, KEYWORD_HASH_MULTIPLIER);
+            assert!(words[slot] == 0, "keyword hash collision");
             words[slot] = word;
             kinds[slot] = kind;
             keywords = rest;
@@ -1688,7 +1672,7 @@ impl KeywordTable {
         // any bytes loaded from the source after the identifier.
         let word = ((continuation << 8) | u64::from(u32::from(first))) << ((8 - len) * 8);
 
-        let index = (word.wrapping_mul(KEYWORD_HASH_MULTIPLIER) >> 58) as usize;
+        let index = keywords::slot(word, KEYWORD_HASH_MULTIPLIER);
         if word == self.words[index] {
             self.kinds[index]
         } else {

crates/ruff_python_parser/src/lexer/keywords.rs

@@ -1,12 +1,11 @@
-//! This file is generated by `scripts/generate_keyword_hash.py`.
-
 use ruff_python_ast::token::TokenKind;
 
-use super::KeywordTable;
+use super::{ASCII_IDENTIFIER_CONTINUE, KeywordTable, is_ascii_identifier_start};
 
-pub(super) const KEYWORD_HASH_MULTIPLIER: u64 = 0x3583_665f_2997_1427;
+pub(super) static KEYWORD_TABLE: KeywordTable = KeywordTable::new(KEYWORDS);
+pub(super) const KEYWORD_HASH_MULTIPLIER: u64 = find_multiplier();
 
-pub(super) static KEYWORD_TABLE: KeywordTable = KeywordTable::new(&[
+const KEYWORDS: &[(&[u8], TokenKind)] = &[
     (b"False", TokenKind::False),
     (b"None", TokenKind::None),
     (b"True", TokenKind::True),
@@ -46,4 +45,99 @@ pub(super) static KEYWORD_TABLE: KeywordTable = KeywordTable::new(&[
     (b"while", TokenKind::While),
     (b"with", TokenKind::With),
     (b"yield", TokenKind::Yield),
-]);
+];
+
+/// Packs a keyword in little-endian order at the high end of a word.
+pub(super) const fn pack(keyword: &[u8]) -> u64 {
+    assert!(
+        keyword.len() >= 2 && keyword.len() <= 8,
+        "keywords must contain two to eight ASCII bytes"
+    );
+    assert!(
+        is_ascii_identifier_start(keyword[0] as char),
+        "keywords must be ASCII identifiers"
+    );
+    let mut word = 0;
+    let mut i = 0;
+    while i < keyword.len() {
+        let byte = keyword[i];
+        assert!(
+            ASCII_IDENTIFIER_CONTINUE[byte as usize],
+            "keywords must be ASCII identifiers"
+        );
+        word = (word >> 8) | ((byte as u64) << 56);
+        i += 1;
+    }
+    word
+}
+
+/// Uses the high six bits of the product to select one of 64 slots.
+pub(super) const fn slot(word: u64, multiplier: u64) -> usize {
+    (word.wrapping_mul(multiplier) >> 58) as usize
+}
+
+/// An `n`-byte keyword ends in `8 - n` zero bytes once packed, so its slot
+/// depends only on the low `n` bytes of the multiplier. Choose those bytes
+/// from low to high. Each new byte leaves all shorter keywords in their slots.
+const fn find_multiplier() -> u64 {
+    assert!(
+        KEYWORDS.len() <= 64,
+        "too many keywords for a 64-slot table"
+    );
+    // Zero pads each group because an ASCII identifier packs to a nonzero word.
+    let mut by_length = [[0; KEYWORDS.len()]; 7];
+    let mut keywords = KEYWORDS;
+    while let Some((&(keyword, _), rest)) = keywords.split_first() {
+        let word = pack(keyword);
+        let group = &mut by_length[keyword.len() - 2];
+        let mut i = 0;
+        while group[i] != 0 {
+            assert!(group[i] != word, "duplicate keyword spelling");
+            i += 1;
+        }
+        group[i] = word;
+        keywords = rest;
+    }
+
+    // If no byte places the whole group, start over with the next odd low byte.
+    let mut seed = 1;
+    'seeds: while seed < 256 {
+        let mut multiplier = seed;
+        seed += 2;
+        let mut occupied = 0;
+        let mut len = 2;
+        while len <= 8 {
+            let mut byte = 0;
+            loop {
+                if byte == 256 {
+                    continue 'seeds;
+                }
+                let candidate = multiplier | (byte << (8 * (len - 1)));
+                if let Some(slots) = place_keywords(&by_length[len - 2], candidate, occupied) {
+                    multiplier = candidate;
+                    occupied = slots;
+                    break;
+                }
+                byte += 1;
+            }
+            len += 1;
+        }
+        return multiplier;
+    }
+    panic!("no multiplier found for the 64-slot keyword table");
+}
+
+/// Tries to place every keyword of this length alongside the shorter keywords.
+const fn place_keywords(mut words: &[u64], multiplier: u64, mut occupied: u64) -> Option<u64> {
+    while let Some((&word, rest)) = words.split_first()
+        && word != 0
+    {
+        let bit = 1 << slot(word, multiplier);
+        if occupied & bit != 0 {
+            return None;
+        }
+        occupied |= bit;
+        words = rest;
+    }
+    Some(occupied)
+}

scripts/generate_keyword_hash.py

@@ -1,120 +0,0 @@
-# /// script
-# requires-python = ">=3.12"
-# dependencies = []
-# ///
-
-"""Generate the lexer's keyword table and collision-free hash multiplier.
-
-Edit KEYWORDS below, then run `uv run scripts/generate_keyword_hash.py`.
-The generated Rust data is written to `crates/ruff_python_parser/src/lexer/keywords.rs`.
-"""
-
-from __future__ import annotations
-
-import random
-from pathlib import Path
-
-OUTPUT_PATH = (
-    Path(__file__).resolve().parents[1]
-    / "crates/ruff_python_parser/src/lexer/keywords.rs"
-)
-U64_MASK = (1 << 64) - 1
-
-# Both hard and soft keywords receive distinct token kinds in the lexer.
-KEYWORDS = [
-    ("False", "False"),
-    ("None", "None"),
-    ("True", "True"),
-    ("and", "And"),
-    ("as", "As"),
-    ("assert", "Assert"),
-    ("async", "Async"),
-    ("await", "Await"),
-    ("break", "Break"),
-    ("case", "Case"),
-    ("class", "Class"),
-    ("continue", "Continue"),
-    ("def", "Def"),
-    ("del", "Del"),
-    ("elif", "Elif"),
-    ("else", "Else"),
-    ("except", "Except"),
-    ("finally", "Finally"),
-    ("for", "For"),
-    ("from", "From"),
-    ("global", "Global"),
-    ("if", "If"),
-    ("import", "Import"),
-    ("in", "In"),
-    ("is", "Is"),
-    ("lazy", "Lazy"),
-    ("lambda", "Lambda"),
-    ("match", "Match"),
-    ("nonlocal", "Nonlocal"),
-    ("not", "Not"),
-    ("or", "Or"),
-    ("pass", "Pass"),
-    ("raise", "Raise"),
-    ("return", "Return"),
-    ("try", "Try"),
-    ("type", "Type"),
-    ("while", "While"),
-    ("with", "With"),
-    ("yield", "Yield"),
-]
-
-
-def find_multiplier(words: list[int]) -> int:
-    """Find a multiplier whose high six product bits uniquely identify each word."""
-    random_source = random.Random(0)
-    for _ in range(10_000_000):
-        multiplier = random_source.getrandbits(64) | 1
-        slots = set()
-        for word in words:
-            # Match Rust's wrapping_mul followed by a shift of 58 bits.
-            slot = ((word * multiplier) & U64_MASK) >> 58
-            if slot in slots:
-                break
-            slots.add(slot)
-        else:
-            return multiplier
-
-    raise SystemExit("No multiplier found; the keyword hash may need a larger table.")
-
-
-def main() -> None:
-    keywords = [keyword for keyword, _ in KEYWORDS]
-    if len(set(keywords)) != len(keywords):
-        raise SystemExit("The keyword list contains duplicate spellings.")
-    if any(
-        not (keyword.isascii() and keyword.isidentifier() and 2 <= len(keyword) <= 8)
-        for keyword in keywords
-    ):
-        raise SystemExit("Keywords must be ASCII identifiers of two to eight bytes.")
-
-    # Match KeywordTable::new: little-endian bytes aligned to the high end of a u64.
-    words = [
-        int.from_bytes(keyword.encode("ascii"), "little") << ((8 - len(keyword)) * 8)
-        for keyword in keywords
-    ]
-    multiplier = find_multiplier(words)
-    entries = "".join(
-        f'    (b"{keyword}", TokenKind::{kind}),\n' for keyword, kind in KEYWORDS
-    )
-    OUTPUT_PATH.write_text(
-        "//! This file is generated by `scripts/generate_keyword_hash.py`.\n"
-        "\n"
-        "use ruff_python_ast::token::TokenKind;\n"
-        "\n"
-        "use super::KeywordTable;\n"
-        "\n"
-        f"pub(super) const KEYWORD_HASH_MULTIPLIER: u64 = 0x{multiplier:_x};\n"
-        "\n"
-        "pub(super) static KEYWORD_TABLE: KeywordTable = KeywordTable::new(&[\n"
-        f"{entries}"
-        "]);\n"
-    )
-
-
-if __name__ == "__main__":
-    main()

scripts/generate_keyword_hash.py.lock

@@ -1,7 +0,0 @@
-version = 1
-revision = 5
-requires-python = ">=3.12"
-
-[options]
-exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
-exclude-newer-span = "P7D"