[ruby/prism] Add checksum

CRuby is now hashing the contents of a file and attaching that to iseqs in order to properly determine if a file has changed since it was compiled. This is used in a couple places, but namely it is used for error_highlight to make sure it isn't highlighting stale locations. The hashing algorithm doesn't actually matter since it's just used to compare across parses, so as long as it is consistent it's okay. I wanted something fast and standard, so this commit implements the XXH3_64 algorithm from the spec. Noteably it uses the default secret and a seed of 0 to make things even simpler. With NEON and SSE2 implementations, this thing flies quite a bit faster than the upstream FNV-1a implementation. I figured this could be useful for other consumers as well, in particular for tools that generate some kind of metadata around files (type checkers, linters, etc). Eventually a nice extension may be to hash a minimal version of the contents (i.e., with context-insensitive whitespace removed) so that changes to docs or indentation do not change the hash value. If you're using the C API, you are only charged for this function if you actually call it since it is computed lazily. For everyone else it is automatically called, but truthfully it is so fast that the difference is completely negligible. https://github.com/ruby/prism/commit/6b7c5773eb
Merged: https://github.com/ruby/ruby/pull/19429
2026-10-28 16:36:20 +08:00 · 2026-10-19 17:30:05 -04:00 · 2026-10-19 21:30:40 +00:00
1 parent 6ca4a12032
commit ae6ef1bfb5
@@ -34,15 +34,18 @@ module Ruby::Prism
    # empty when the source is created by the parser and filled in afterward,
    # because the comments themselves refer to the source.
    #
-    #: (String source, Integer start_line, Array[Integer] offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges) -> Source
-    def self.for(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [])
+    # The checksum parameter is the value described by the attribute of the same
+    # name. It is nil when the source is not created by the parser.
+    #
+    #: (String source, Integer start_line, Array[Integer] offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges, ?Integer? checksum) -> Source
+    def self.for(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [], checksum = nil)
      if source.ascii_only?
-        ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
+        ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges, checksum)
      elsif source.encoding == Encoding::BINARY
        source.force_encoding(Encoding::UTF_8)

        if source.valid_encoding?
-          new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
+          new(source, start_line, offsets, comments, comment_attachments, comment_ranges, checksum)
        else
          # This is an extremely niche use case where the file is marked as
          # binary, contains multi-byte characters, and those characters are not
@@ -51,10 +54,10 @@ module Ruby::Prism
          # problems when asking for code units, but it appears to be the
          # cleanest solution at the moment.
          source.force_encoding(Encoding::BINARY)
-          ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
+          ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges, checksum)
        end
      else
-        new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
+        new(source, start_line, offsets, comments, comment_attachments, comment_ranges, checksum)
      end
    end

@@ -64,6 +67,10 @@ module Ruby::Prism
    # The line number where this source starts.
    attr_reader :start_line #: Integer

+    # The checksum of the source code which can be used to quickly determine if
+    # the source has changed.
+    attr_reader :checksum #: Integer?
+
    # The list of comments that were encountered during parsing. When the source
    # is parsed with the attach_comments option, these are sorted by start
    # offset.
@@ -95,10 +102,11 @@ module Ruby::Prism
    # parameter can be either an Array of Integer byte offsets or a packed
    # binary string of uint32_t values (from the C extension).
    #
-    #: (String source, Integer start_line, Array[Integer] | String offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges) -> void
-    def initialize(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [])
+    #: (String source, Integer start_line, Array[Integer] | String offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges, ?Integer? checksum) -> void
+    def initialize(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [], checksum = nil)
      @source = source
      @start_line = start_line
+      @checksum = checksum
      @offsets = offsets
      @comments = comments
      @comment_attachments = comment_attachments
@@ -112,6 +120,13 @@ module Ruby::Prism
      @start_line = start_line
    end

+    # Replace the value of checksum with the given value.
+    #
+    #: (Integer checksum) -> void
+    def replace_checksum(checksum)
+      @checksum = checksum
+    end
+
    # Replace the value of offsets with the given value.
    #
    #: (Array[Integer] offsets) -> void
@@ -186,6 +186,15 @@ pm_buffer_append_double(pm_buffer_t *buffer, double value) {
    pm_buffer_append(buffer, source, sizeof(double));
 }

+/**
+ * Append a 64-bit unsigned integer to the buffer in native endian order.
+ */
+void
+pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value) {
+    const void *source = &value;
+    pm_buffer_append(buffer, source, sizeof(uint64_t));
+}
+
 /**
 * Append a unicode codepoint to the buffer.
 */
@@ -4,6 +4,8 @@
 #ifndef PRISM_COMPILER_ACCEL_H
 #define PRISM_COMPILER_ACCEL_H

+#include "prism/compiler/endianness.h"
+
 /**
 * Platform detection for SIMD/fast-path implementations. At most one of these
 * macros is defined, selecting the best available vectorization strategy.
@@ -17,7 +19,7 @@
 #   define PRISM_HAS_NEON
 #elif (defined(__x86_64__) && defined(__SSE2__)) || (defined(_MSC_VER) && defined(_M_X64))
 #   define PRISM_HAS_SSE2
-#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#elif defined(PRISM_LITTLE_ENDIAN_COMPTIME)
 #   define PRISM_HAS_SWAR
 #endif

@@ -0,0 +1,46 @@
+/**
+ * @file compiler/bswap.h
+ */
+#ifndef PRISM_COMPILER_BSWAP_H
+#define PRISM_COMPILER_BSWAP_H
+
+/**
+ * Byte swap utilities for 32-bit and 64-bit integers.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#define pm_bswap32(x_) __builtin_bswap32((x_))
+#define pm_bswap64(x_) __builtin_bswap64((x_))
+#elif defined(_MSC_VER)
+#include <stdlib.h>
+
+#define pm_bswap32(x_) _byteswap_ulong((x_))
+#define pm_bswap64(x_) _byteswap_uint64((x_))
+#else
+#include <stdint.h>
+
+static uint32_t
+pm_bswap32(uint32_t x) {
+    return (
+      ((x >> 24) & 0x000000FFUL) |
+      ((x << 24) & 0xFF000000UL) |
+      ((x >> 8)  & 0x0000FF00UL) |
+      ((x << 8)  & 0x00FF0000UL)
+    );
+}
+
+static uint64_t
+pm_bswap64(uint64_t x) {
+    return (
+      ((x >> 56) & 0x00000000000000FFULL) |
+      ((x << 56) & 0xFF00000000000000ULL) |
+      ((x >> 40) & 0x000000000000FF00ULL) |
+      ((x << 40) & 0x00FF000000000000ULL) |
+      ((x >> 24) & 0x0000000000FF0000ULL) |
+      ((x << 24) & 0x0000FF0000000000ULL) |
+      ((x >> 8)  & 0x00000000FF000000ULL) |
+      ((x << 8)  & 0x000000FF00000000ULL)
+    );
+}
+#endif
+
+#endif
@@ -0,0 +1,34 @@
+/**
+ * @file compiler/endianness.h
+ */
+#ifndef PRISM_COMPILER_ENDIANNESS_H
+#define PRISM_COMPILER_ENDIANNESS_H
+
+/**
+ * Detect endianness at compile-time through the use of predefined macros.
+ */
+#if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define PRISM_LITTLE_ENDIAN_COMPTIME
+#endif
+
+/**
+ * Determine endianness at runtime, either through predefined macros or through
+ * a runtime check that is usually folded.
+ */
+#if defined(PRISM_LITTLE_ENDIAN_COMPTIME)
+#define PRISM_LITTLE_ENDIAN_RUNTIME 1
+#elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define PRISM_LITTLE_ENDIAN_RUNTIME 0
+#else
+#include <stdint.h>
+
+static inline int
+pm_little_endian(void) {
+    const union { uint32_t u; uint8_t c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+#define PRISM_LITTLE_ENDIAN_RUNTIME pm_little_endian()
+#endif
+
+#endif
@@ -983,8 +983,9 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o

    result_t result = check_raise_error_option(parser, options, path_encoding);
    if (result.type == RESULT_OK) {
-        /* Update the Source object with the correct encoding and line offsets,
-         * which are only available after pm_parse() completes. */
+        /* Update the Source object with the correct encoding, line offsets,
+         * and source hash, which are only available after pm_parse()
+         * completes. */
        rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
        rb_enc_associate(source_string, encoding);

@@ -993,6 +994,8 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o
            rb_ary_store(offsets, (long) index, ULONG2NUM(line_offsets->offsets[index]));
        }

+        rb_funcall(source, rb_intern("replace_checksum"), 1, ULL2NUM(pm_parser_checksum(parser)));
+
        if (pm_options_freeze(options)) {
            rb_obj_freeze(source_string);
            rb_obj_freeze(offsets);
@@ -52,6 +52,9 @@ void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
 /* Append a double to the buffer. */
 void pm_buffer_append_double(pm_buffer_t *buffer, double value);

+/* Append a 64-bit unsigned integer to the buffer in native endian order. */
+void pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value);
+
 /* Append a unicode codepoint to the buffer. */
 bool pm_buffer_append_unicode_codepoint(pm_buffer_t *buffer, uint32_t value);

@@ -0,0 +1,16 @@
+#ifndef PRISM_INTERNAL_XXH3_H
+#define PRISM_INTERNAL_XXH3_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * Computes the XXH3 digest of the given input.
+ *
+ * @param input The input data to hash.
+ * @param length The length of the input data.
+ * @return The 64-bit XXH3 hash of the input data.
+ */
+uint64_t pm_xxh3(const uint8_t *input, size_t length);
+
+#endif
@@ -5,8 +5,10 @@
 #include "prism/internal/diagnostic.h"
 #include "prism/internal/encoding.h"
 #include "prism/internal/magic_comments.h"
+#include "prism/internal/xxh3.h"

 #include <stdlib.h>
+#include <string.h>

 /**
 * Register a callback that will be called whenever prism changes the encoding
@@ -131,6 +133,22 @@ pm_parser_data_loc(const pm_parser_t *parser) {
    return &parser->data_loc;
 }

+/**
+ * Returns the checksum of the source code that was parsed.
+ */
+uint64_t
+pm_parser_checksum(const pm_parser_t *parser) {
+    const uint8_t *end = parser->end;
+
+    if (parser->data_loc.length != 0) {
+        const uint8_t *cursor = parser->start + parser->data_loc.start;
+        const uint8_t *newline = memchr(cursor, '\n', (size_t) (end - cursor));
+        if (newline != NULL) end = newline + 1;
+    }
+
+    return pm_xxh3(parser->start, (size_t) (end - parser->start));
+}
+
 /**
 * Returns whether the given parser is continuable, meaning that it could become
 * valid if more input were appended, as opposed to being definitively invalid.
@@ -177,6 +177,27 @@ PRISM_EXPORTED_FUNCTION const pm_line_offset_list_t * pm_parser_line_offsets(con
 */
 PRISM_EXPORTED_FUNCTION const pm_location_t * pm_parser_data_loc(const pm_parser_t *parser) PRISM_NONNULL(1);

+/**
+ * Returns the checksum of the source code that was parsed. The data section
+ * after an __END__ marker is not code, so the hash covers the source only up to
+ * and including the newline that ends the __END__ line. The __END__ marker is
+ * only found by parsing, so this must be called after pm_parse.
+ *
+ * The hash is computed from the source on each call rather than during
+ * parsing, so that parses that never ask for it do not pay for it. This
+ * requires the source to still be alive, and callers that need the value more
+ * than once should keep it.
+ *
+ * The hash is currently the 64-bit variant of XXH3 with a seed of 0. It is not
+ * cryptographic, and the algorithm may change between versions of prism, so
+ * hashes should only be compared against other hashes computed by the same
+ * version of prism.
+ *
+ * @param parser the parser whose checksum we want to get
+ * @returns the checksum of the source code that was parsed
+ */
+PRISM_EXPORTED_FUNCTION uint64_t pm_parser_checksum(const pm_parser_t *parser) PRISM_NONNULL(1);
+
 /**
 * Returns whether the given parser is continuable, meaning that it could become
 * valid if more input were appended, as opposed to being definitively invalid.
@@ -23908,6 +23908,7 @@ pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t s
    pm_serialize_header(buffer);
    pm_serialize_encoding(parser.encoding, buffer);
    pm_buffer_append_varsint(buffer, parser.start_line);
+    pm_buffer_append_uint64(buffer, pm_parser_checksum(&parser));
    pm_serialize_line_offset_list(&parser.line_offsets, buffer);
    pm_serialize_comment_list(&parser.comment_list, buffer);

@@ -96,7 +96,7 @@ pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, VALUE comments,
        offsets = rb_str_new((const char *) line_offsets->offsets, line_offsets->size * sizeof(uint32_t));
    }

-    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 6, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, comment_attachments, comment_ranges);
+    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 7, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, comment_attachments, comment_ranges, ULL2NUM(pm_parser_checksum(parser)));
    if (freeze) rb_obj_freeze(source);

    return source;
@@ -33,9 +33,11 @@ module Ruby::Prism
                       loader.load_header
      encoding =       loader.load_encoding
      start_line =     loader.load_varsint
+      checksum =       loader.load_uint64
      offsets =        loader.load_line_offsets(freeze)

      source.replace_start_line(start_line)
+      source.replace_checksum(checksum)
      source.replace_offsets(offsets)

                       loader.load_comments(source.comments, freeze)
@@ -89,9 +91,11 @@ module Ruby::Prism
      tokens =         loader.load_tokens
      encoding =       loader.load_encoding
      start_line =     loader.load_varsint
+      checksum =       loader.load_uint64
      offsets =        loader.load_line_offsets(freeze)

      source.replace_start_line(start_line)
+      source.replace_checksum(checksum)
      source.replace_offsets(offsets)

                       loader.load_comments(source.comments, freeze)
@@ -134,16 +138,18 @@ module Ruby::Prism
      source = Source.for(input, 1, [])
      loader = Loader.new(source, serialized)

-                   loader.load_header
-                   loader.load_encoding
-      start_line = loader.load_varsint
-      offsets    = loader.load_line_offsets(freeze)
+                    loader.load_header
+                    loader.load_encoding
+      start_line =  loader.load_varsint
+      checksum =    loader.load_uint64
+      offsets =     loader.load_line_offsets(freeze)

      source.replace_start_line(start_line)
+      source.replace_checksum(checksum)
      source.replace_offsets(offsets)

-      result =     loader.load_comments(source.comments, freeze)
-      raise unless loader.eof?
+      result =      loader.load_comments(source.comments, freeze)
+      raise unless  loader.eof?

      source.deep_freeze if freeze
      result
@@ -164,9 +170,11 @@ module Ruby::Prism
                       loader.load_header
      encoding =       loader.load_encoding
      start_line =     loader.load_varsint
+      checksum =       loader.load_uint64
      offsets =        loader.load_line_offsets(freeze)

      source.replace_start_line(start_line)
+      source.replace_checksum(checksum)
      source.replace_offsets(offsets)

                       loader.load_comments(source.comments, freeze)
@@ -531,6 +539,11 @@ module Ruby::Prism
        (io.read(4) or raise).unpack1("L") #: Integer
      end

+      #: () -> Integer
+      def load_uint64
+        (io.read(8) or raise).unpack1("Q") #: Integer
+      end
+
      #: (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node?
      def load_optional_node(constant_pool, encoding, freeze)
        if io.getbyte != 0
@@ -293,6 +293,7 @@ static void
 pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) {
    pm_serialize_encoding(parser->encoding, buffer);
    pm_buffer_append_varsint(buffer, parser->start_line);
+    pm_buffer_append_uint64(buffer, pm_parser_checksum(parser));
    pm_serialize_line_offset_list(&parser->line_offsets, buffer);
 <%- unless Ruby::Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%>
    pm_serialize_comment_list(&parser->comment_list, buffer);
@@ -0,0 +1,292 @@
+/**
+ * An implementation of the XXH3 hash function.
+ *
+ * The reason some of these names do not look quite like the style of the rest
+ * of the code in Prism is that this was adapted straight from the XXH3
+ * specification here:
+ *
+ * https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
+ *
+ * To better understand this code it is recommended to read the XXH3
+ * specification linked above, where you should be able to follow along with
+ * this implementation more easily. Note this only implements the XXH3_64
+ * variant with the default secret and a seed of 0. Additionally it does not
+ * include a lot of the various CPU-specific optimizations that exist in the
+ * reference implementation.
+ */
+
+#include "prism/internal/xxh3.h"
+
+#include "prism/compiler/accel.h"
+#include "prism/compiler/bswap.h"
+#include "prism/compiler/endianness.h"
+
+#include <string.h>
+
+#if defined(PRISM_HAS_NEON) && defined(PRISM_LITTLE_ENDIAN_COMPTIME)
+    #include <arm_neon.h>
+    #define XXH3_NEON
+#elif defined(PRISM_HAS_SSE2)
+    #include <emmintrin.h>
+    #define XXH3_SSE2
+#endif
+
+#if defined(__SIZEOF_INT128__)
+static uint64_t
+XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
+    __uint128_t x = (__uint128_t) lhs * (__uint128_t) rhs;
+    return ((uint64_t) (x & 0xFFFFFFFFFFFFFFFFULL)) ^ ((uint64_t) (x >> 64));
+}
+#else
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
+#include <intrin.h>
+#endif
+
+static uint64_t
+XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
+    uint64_t lo, hi;
+#if defined(_MSC_VER) && defined(_M_X64)
+    lo = _umul128(lhs, rhs, &hi);
+#elif defined(_MSC_VER) && defined(_M_ARM64)
+    hi = __umulh(lhs, rhs);
+    lo = lhs * rhs;
+#else
+    uint64_t lo_lo = (lhs & 0xFFFFFFFF) * (rhs & 0xFFFFFFFF);
+    uint64_t hi_lo = (lhs >> 32) * (rhs & 0xFFFFFFFF);
+    uint64_t lo_hi = (lhs & 0xFFFFFFFF) * (rhs >> 32);
+    uint64_t hi_hi = (lhs >> 32) * (rhs >> 32);
+
+    uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+    hi = hi_hi + (hi_lo >> 32) + (cross >> 32);
+    lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+#endif
+    return lo ^ hi;
+}
+#endif
+
+static uint32_t
+read32(const void* ptr) {
+    uint32_t val;
+    memcpy(&val, ptr, sizeof(val));
+    if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
+        val = pm_bswap32(val);
+    }
+    return val;
+}
+
+static uint64_t
+read64(const void* ptr) {
+    uint64_t val;
+    memcpy(&val, ptr, sizeof(val));
+    if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
+        val = pm_bswap64(val);
+    }
+    return val;
+}
+
+static const uint64_t PRIME32_1 = 0x9E3779B1U;
+static const uint64_t PRIME32_2 = 0x85EBCA77U;
+static const uint64_t PRIME32_3 = 0xC2B2AE3DU;
+static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
+static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
+static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
+static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
+static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;
+static const uint64_t PRIME_MX1 = 0x165667919E3779F9ULL;
+static const uint64_t PRIME_MX2 = 0x9FB21C651E98DF25ULL;
+
+static const uint8_t secret[192] = {
+    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+static uint64_t
+avalanche(uint64_t x) {
+    x ^= (x >> 37);
+    x *= PRIME_MX1;
+    x ^= (x >> 32);
+    return x;
+}
+
+static uint64_t
+avalanche_XXH64(uint64_t x) {
+    x ^= (x >> 33);
+    x *= PRIME64_2;
+    x ^= (x >> 29);
+    x *= PRIME64_3;
+    x ^= (x >> 32);
+    return x;
+}
+
+static uint64_t
+XXH3_64_empty(void) {
+    return avalanche_XXH64(read64(secret + 56) ^ read64(secret + 64));
+}
+
+static uint64_t
+XXH3_64_1to3(const uint8_t *input, size_t length) {
+    uint32_t combined = (uint32_t) input[length - 1] | ((uint32_t) length << 8) | ((uint32_t) input[0] << 16) | ((uint32_t) input[length >> 1] << 24);
+    return avalanche_XXH64((uint64_t) (read32(secret) ^ read32(secret + 4)) ^ (uint64_t) combined);
+}
+
+static uint64_t
+XXH3_64_4to8(const uint8_t *input, size_t length) {
+#define rot(x_, r_) (((x_) << (r_)) | ((x_) >> (64 - (r_))))
+
+    uint64_t combined = (uint64_t) read32(input + length - 4) | ((uint64_t) read32(input) << 32);
+    uint64_t value = (read64(secret + 8) ^ read64(secret + 16)) ^ combined;
+    value ^= rot(value, 49) ^ rot(value, 24);
+    value *= PRIME_MX2;
+    value ^= ((value >> 35) + length);
+    value *= PRIME_MX2;
+    value ^= (value >> 28);
+    return value;
+
+#undef rot
+}
+
+static uint64_t
+XXH3_64_9to16(const uint8_t *input, size_t length) {
+    uint64_t lo = (read64(secret + 24) ^ read64(secret + 32)) ^ read64(input);
+    uint64_t hi = (read64(secret + 40) ^ read64(secret + 48)) ^ read64(input + length - 8);
+    return avalanche(length + pm_bswap64(lo) + hi + XXH3_mul128_fold64(lo, hi));
+}
+
+static uint64_t
+XXH3_64_17to240(const uint8_t *input, size_t length) {
+#define mix(data_, offset_) XXH3_mul128_fold64(read64((data_)) ^ read64(secret + (offset_)), read64((data_) + 8) ^ read64(secret + (offset_) + 8))
+
+    uint64_t acc = length * PRIME64_1;
+
+    if (length <= 128) {
+        size_t idx = (length - 1) / 32;
+        do {
+            acc += mix(input + idx * 16, idx * 32) + mix(input + length - idx * 16 - 16, idx * 32 + 16);
+        } while (idx--);
+    } else {
+        for (size_t idx = 0; idx < 8; idx++) {
+            acc += mix(input + idx * 16, idx * 16);
+        }
+        acc = avalanche(acc);
+        for (size_t idx = 8; idx < length >> 4; idx++) {
+            acc += mix(input + idx * 16, (idx - 8) * 16 + 3);
+        }
+        acc += mix(input + length - 16, 119);
+    }
+
+    return avalanche(acc);
+
+#undef mix
+}
+
+static void
+accumulate(uint64_t acc[8], const uint8_t *stripe, size_t secret_offset) {
+#if defined(XXH3_NEON)
+    for (int idx = 0; idx < 8; idx += 2) {
+        uint64x2_t data = vreinterpretq_u64_u8(vld1q_u8(stripe + 8 * idx));
+        uint64x2_t value = veorq_u64(data, vreinterpretq_u64_u8(vld1q_u8(secret + secret_offset + 8 * idx)));
+        uint64x2_t product = vmull_u32(vmovn_u64(value), vshrn_n_u64(value, 32));
+        vst1q_u64(acc + idx, vaddq_u64(vld1q_u64(acc + idx), vaddq_u64(vextq_u64(data, data, 1), product)));
+    }
+#elif defined(XXH3_SSE2)
+    for (int idx = 0; idx < 8; idx += 2) {
+        __m128i data = _mm_loadu_si128((const __m128i *) (stripe + 8 * idx));
+        __m128i value = _mm_xor_si128(data, _mm_loadu_si128((const __m128i *) (secret + secret_offset + 8 * idx)));
+        __m128i product = _mm_mul_epu32(value, _mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)));
+        _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_loadu_si128((const __m128i *) (acc + idx)), _mm_add_epi64(_mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)), product)));
+    }
+#else
+    for (int idx = 0; idx < 8; idx++) {
+        uint64_t data = read64(stripe + 8 * idx);
+        uint64_t value = data ^ read64(secret + secret_offset + 8 * idx);
+        acc[idx ^ 1] = acc[idx ^ 1] + data;
+        acc[idx] += (value & 0xFFFFFFFF) * (value >> 32);
+    }
+#endif
+}
+
+static uint64_t
+XXH3_64_large(const uint8_t *input, size_t length) {
+    const size_t secret_length = sizeof(secret);
+    const size_t stripes_per_block = (secret_length - 64) / 8;
+    const size_t block_size = 64 * stripes_per_block;
+    const size_t block_count = (length - 1) / block_size;
+    const size_t last_offset = block_count * block_size;
+
+    uint64_t acc[8] = {
+        PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
+        PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1
+    };
+
+    for (size_t block_idx = 0; block_idx < block_count; block_idx++) {
+        const uint8_t *block = input + block_idx * block_size;
+
+        for (size_t stripe_idx = 0; stripe_idx < stripes_per_block; stripe_idx++) {
+            accumulate(acc, block + stripe_idx * 64, stripe_idx * 8);
+        }
+
+#if defined(XXH3_NEON)
+        const uint32x2_t prime = vdup_n_u32((uint32_t) PRIME32_1);
+        for (size_t idx = 0; idx < 8; idx += 2) {
+            uint64x2_t value = vld1q_u64(acc + idx);
+            value = veorq_u64(value, vshrq_n_u64(value, 47));
+            value = veorq_u64(value, vreinterpretq_u64_u8(vld1q_u8(secret + secret_length - 64 + idx * 8)));
+            vst1q_u64(acc + idx, vaddq_u64(vmull_u32(vmovn_u64(value), prime), vshlq_n_u64(vmull_u32(vshrn_n_u64(value, 32), prime), 32)));
+        }
+#elif defined(XXH3_SSE2)
+        const __m128i prime = _mm_set1_epi32((int) (uint32_t) PRIME32_1);
+        for (size_t idx = 0; idx < 8; idx += 2) {
+            __m128i value = _mm_loadu_si128((const __m128i *) (acc + idx));
+            value = _mm_xor_si128(value, _mm_srli_epi64(value, 47));
+            value = _mm_xor_si128(value, _mm_loadu_si128((const __m128i *) (secret + secret_length - 64 + idx * 8)));
+            _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_mul_epu32(value, prime), _mm_slli_epi64(_mm_mul_epu32(_mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)), prime), 32)));
+        }
+#else
+        for (size_t idx = 0; idx < 8; idx++) {
+            acc[idx] ^= (acc[idx] >> 47);
+            acc[idx] ^= read64(secret + secret_length - 64 + idx * 8);
+            acc[idx] *= PRIME32_1;
+        }
+#endif
+    }
+
+    for (size_t stripe_idx = 0; stripe_idx < (length - last_offset - 1) / 64; stripe_idx++) {
+        accumulate(acc, input + last_offset + stripe_idx * 64, stripe_idx * 8);
+    }
+
+    accumulate(acc, input + length - 64, secret_length - 71);
+
+    uint64_t result = (uint64_t) length * PRIME64_1;
+    for (size_t idx = 0; idx < 4; idx++) {
+        result += XXH3_mul128_fold64(acc[idx * 2] ^ read64(secret + 11 + 16 * idx), acc[idx * 2 + 1] ^ read64(secret + 11 + 16 * idx + 8));
+    }
+
+    return avalanche(result);
+}
+
+uint64_t
+pm_xxh3(const uint8_t *input, size_t length) {
+    if (length == 0) {
+        return XXH3_64_empty();
+    } else if (length <= 3) {
+        return XXH3_64_1to3(input, length);
+    } else if (length <= 8) {
+        return XXH3_64_4to8(input, length);
+    } else if (length <= 16) {
+        return XXH3_64_9to16(input, length);
+    } else if (length <= 240) {
+        return XXH3_64_17to240(input, length);
+    } else {
+        return XXH3_64_large(input, length);
+    }
+}
@@ -47,5 +47,54 @@ module Ruby::Prism
      error = assert_raise(ArgumentError) { source.byte_offset(9, 0) }
      assert_equal "line 9 is out of range", error.message
    end
+
+    # These expectations generated via xxhsum -H3.
+    def test_checksum
+      assert_equal 0x2d06800538d394c2, Ruby::Prism.parse("").source.checksum
+
+      {
+        2 => 0x369242a646a19333,
+        6 => 0x8a7787871647e141,
+        12 => 0x6d13ee7fa0a78768,
+        100 => 0x411d9368f9c30e07,
+        200 => 0xac2bd404bce6c995,
+        5000 => 0x86bab6aca5d8f5e9
+      }.each do |length, expected|
+        assert_equal expected, Ruby::Prism.parse("a" * length).source.checksum, "length #{length}"
+      end
+    end
+
+    def test_checksum_excludes_data
+      expected = Ruby::Prism.parse("puts 1\n__END__\n").source.checksum
+      assert_equal 0x3326de5be62ee9e4, expected
+
+      assert_equal expected, Ruby::Prism.parse("puts 1\n__END__\ndata\n").source.checksum
+      assert_equal expected, Ruby::Prism.parse("puts 1\n__END__\nother data").source.checksum
+
+      refute_equal expected, Ruby::Prism.parse("puts 1\n").source.checksum
+      refute_equal expected, Ruby::Prism.parse("puts 2\n__END__\n").source.checksum
+    end
+
+    def test_checksum_across_apis
+      source = "# comment\nputs 1\n__END__\ndata\n"
+      expected = Ruby::Prism.parse(source).source.checksum
+
+      assert_equal expected, Ruby::Prism.parse(source, freeze: true).source.checksum
+      assert_equal expected, Ruby::Prism.lex(source).source.checksum
+      assert_equal expected, Ruby::Prism.parse_lex(source).source.checksum
+      if !ENV["PRISM_BUILD_MINIMAL"]
+        assert_equal expected, Ruby::Prism.load(source, Ruby::Prism.dump(source)).source.checksum
+      end
+      assert_equal expected, Ruby::Prism.parse_stream(StringIO.new(source)).source.checksum
+
+      comment = Ruby::Prism.parse_comments(source).first
+      assert_equal expected, comment.location.send(:source).checksum
+    end
+
+    def test_checksum_without_parser
+      assert_nil Source.for("puts 1", 1, [0]).checksum
+      assert_equal 0x1234, Source.for("puts 1", 1, [0], [], [], [], 0x1234).checksum
+      assert_equal 0x1234, Source.for("puts é", 1, [0], [], [], [], 0x1234).checksum
+    end
  end
 end