diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index a9bc34140..c7fc88352 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -51,7 +51,31 @@ jobs: # The benchmarks are linked against libpython through the pyo3 dev-dependency. - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - name: Restore the machine main last measured on + # A recording that cannot be fetched is read as "none yet", which the step + # below treats as nothing to disagree with. Failing here instead would + # turn a cache outage into a red benchmark job. + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codspeed-environment-reference.json + key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} + restore-keys: codspeed-environment-${{ runner.os }}-${{ runner.arch }}- + + - name: Name the machine this run drew + id: environment + shell: bash + run: | + reference=build/codspeed-environment-reference.json + if [ "$GITHUB_REF" = refs/heads/main ]; then + args=(--record "$reference") + else + args=(--record build/codspeed-environment.json --reference "$reference") + fi + python3 scripts/codspeed-environment.py "${args[@]}" --github-output + - name: Restore cache + if: ${{ steps.environment.outputs.match != 'false' }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ env.CACHE_PATHS }} @@ -60,9 +84,11 @@ jobs: continue-on-error: true - name: Install cargo-codspeed + if: ${{ steps.environment.outputs.match != 'false' }} run: cargo install cargo-codspeed --locked --version ^5 - name: Build the benchmark targets + if: ${{ steps.environment.outputs.match != 'false' }} run: cargo codspeed build --locked --measurement-mode simulation -p rustpython -p rustpython-sre_engine - name: Save cache @@ -74,7 +100,15 @@ jobs: continue-on-error: true - name: Run the benchmarks + if: ${{ steps.environment.outputs.match != 'false' }} uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 with: mode: simulation run: cargo codspeed run -p rustpython -p rustpython-sre_engine + + - name: Record the machine this baseline was measured on + if: ${{ success() && github.ref == 'refs/heads/main' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codspeed-environment-reference.json + key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} diff --git a/.gitignore b/.gitignore index cb8548877..fdacf4f23 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ wasm-pack.log flame-graph.html flame.txt flamescope.json +/build/codspeed-environment*.json /wapm.lock /wapm_packages /.cargo/config diff --git a/scripts/codspeed-environment.py b/scripts/codspeed-environment.py new file mode 100755 index 000000000..2c018e136 --- /dev/null +++ b/scripts/codspeed-environment.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Name the machine a CodSpeed measurement was taken on, and hold a branch to it. + +CodSpeed reports a run as a change against a base run, and the runner uploads a +description of the machine beside the profile: `SystemInfo` in +`CodSpeedHQ/runner` carries the OS and its version, the architecture, the CPU's +vendor id, brand and flags, the physical core count and the installed memory. +When the two runs disagree on it the report says "different runtime +environments detected" and the numbers below that line are not a comparison of +two trees. A hosted runner pool hands out whatever machine is free, so two runs +of the same workflow do not share one by construction. + +A simulated measurement does not move with how fast the machine is -- the cache +geometry the runner hands Valgrind is fixed, and two runs whose wall clock +differs by half still report the same counts. It does move with what the +machine makes the process execute: glibc resolves `memcpy`, `memset` and the +string routines through IFUNC selectors keyed on CPU flags, so one binary has +one instruction stream per selection. The toolchain is the same class of input +one level up, since a new stable rustc emits different code. So is the CPython +these benchmarks are linked against: `benches/execution.rs` and +`benches/microbenchmarks.rs` run each program under CPython through pyo3 as well +as under RustPython, and the runner's default interpreter moves with its image. + +The rule this enforces: `main` defines the environment. Every run on it records +what it measured on, and a run on any other ref measures only when it matches +that recording. Anything else is skipped rather than uploaded, because an +uploaded cross-environment run is published as a regression of the branch that +happened to draw the other machine. A recording that cannot be read is +"anything else" too: an unknown machine is not a matching one, and reading it as +one would leave the guard open exactly while it has nothing to check against. + + codspeed-environment.py --record build/codspeed-environment.json + codspeed-environment.py --record CURRENT --reference REFERENCE --github-output + +Exits 0 whether or not the environments match; the verdict is the `match` +output, so the caller decides what a mismatch costs. +""" + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import subprocess +import sys + +# The fields a comparison is made over. `SystemInfo` also carries the host name +# and the invoking user, which a hosted runner changes on every run and which +# therefore cannot be part of a rule that any run is expected to satisfy. +COMPARED_FIELDS = ( + "os", + "arch", + "cpu_vendor_id", + "cpu_brand", + "cpu_cores", + "total_memory_gb", + "cpu_flags", + "rustc", + "libc", + "python", +) + + +def _command_output(argv): + try: + out = subprocess.run( + argv, capture_output=True, text=True, check=False, timeout=60 + ) + except (OSError, subprocess.SubprocessError): + return "" + # A probe that fails contributes nothing rather than an error string, so a + # field absent on one platform stays absent instead of becoming a value. + return out.stdout.strip() if out.returncode == 0 else "" + + +def _os_release(): + """The distribution and its version, as `SupportedOs` spells them.""" + try: + with open("/etc/os-release", encoding="utf-8") as handle: + fields = dict( + line.rstrip("\n").split("=", 1) + for line in handle + if "=" in line and not line.startswith("#") + ) + except OSError: + return f"{platform.system()} {platform.release()}" + name = fields.get("ID", "linux").strip('"') + version = fields.get("VERSION_ID", "").strip('"') + return f"{name} {version}".strip() + + +def _linux_cpu(): + with open("/proc/cpuinfo", encoding="utf-8") as handle: + return parse_cpuinfo(handle.read()) + + +def parse_cpuinfo(text): + blocks = [block for block in text.split("\n\n") if block.strip()] + + def field(block, name): + match = re.search(rf"^{name}\s*:\s*(.*)$", block, re.MULTILINE) + return match.group(1).strip() if match else "" + + first = blocks[0] if blocks else "" + # Count distinct physical cores the way `sysinfo::System::physical_core_count` + # does: one entry per (socket, core), so a hyperthreaded pair counts once. + # aarch64 names neither, and the file ends in a machine-wide block that no + # `processor` line introduces, so the fallback counts those lines instead. + cores = {(field(block, "physical id"), field(block, "core id")) for block in blocks} + cores.discard(("", "")) + logical = len(re.findall(r"^processor\s*:", text, re.MULTILINE)) + # x86_64 spells the CPU's identity one way and aarch64 another; a field + # missing on one architecture is constant there and carries no signal. + vendor = field(first, "vendor_id") or field(first, "CPU implementer") + brand = ( + field(first, "model name") + or " ".join( + field(first, name) for name in ("CPU part", "CPU variant", "CPU revision") + ).strip() + ) + flags = field(first, "flags") or field(first, "Features") + return { + "cpu_vendor_id": vendor, + "cpu_brand": brand, + "cpu_cores": len(cores) or logical, + "cpu_flags": sorted(flags.split()), + } + + +def _darwin_cpu(): + def sysctl(name): + return _command_output(["sysctl", "-n", name]) + + flags = " ".join( + sysctl(name) for name in ("machdep.cpu.features", "hw.optional.arm.caps") + ) + return { + "cpu_vendor_id": sysctl("machdep.cpu.vendor") or platform.machine(), + "cpu_brand": sysctl("machdep.cpu.brand_string"), + "cpu_cores": int(sysctl("hw.physicalcpu") or 0), + "cpu_flags": sorted(flags.split()), + } + + +def _total_memory_gb(): + """Installed memory in whole GiB, rounded up as `sysinfo` reports it.""" + try: + with open("/proc/meminfo", encoding="utf-8") as handle: + for line in handle: + if line.startswith("MemTotal:"): + return math.ceil(int(line.split()[1]) * 1024 / 1024**3) + except OSError: + pass + size = _command_output(["sysctl", "-n", "hw.memsize"]) + return math.ceil(int(size) / 1024**3) if size.isdigit() else 0 + + +def _libc_version(): + """The C library whose IFUNC selectors decide the string routines.""" + version = _command_output(["ldd", "--version"]) + return version.splitlines()[0].strip() if version else "" + + +def _python_version(): + """The interpreter the benchmarks measure beside RustPython. + + Read from the process running this script rather than from a subprocess, so + it names the `python3` the workflow put on PATH -- which is the one the + pyo3 build resolves libpython from. + """ + return f"{platform.python_implementation()} {platform.python_version()}" + + +def collect(): + cpu = _linux_cpu() if sys.platform.startswith("linux") else _darwin_cpu() + environment = { + "os": _os_release(), + "arch": platform.machine(), + "total_memory_gb": _total_memory_gb(), + "rustc": _command_output(["rustc", "--version"]), + "libc": _libc_version(), + "python": _python_version(), + **cpu, + } + environment["digest"] = digest(environment) + return environment + + +def digest(environment): + payload = json.dumps( + {name: environment.get(name) for name in COMPARED_FIELDS}, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16] + + +def differences(current, reference): + return [ + (name, reference.get(name), current.get(name)) + for name in COMPARED_FIELDS + if reference.get(name) != current.get(name) + ] + + +def _render(environment): + for name in COMPARED_FIELDS: + value = environment.get(name) + if name == "cpu_flags": + value = f"{len(value)} flags" + print(f" {name:<17}{value}") + print(f" {'digest':<17}{environment['digest']}") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--record", metavar="PATH", help="write this machine's environment as JSON" + ) + parser.add_argument( + "--reference", + metavar="PATH", + help="compare against a previously recorded environment; a file that" + " cannot be read leaves the machine unidentified, which is not a match", + ) + parser.add_argument( + "--github-output", + action="store_true", + help="append `match` and `digest` to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + + current = collect() + print("Benchmark environment:") + _render(current) + + match = True + annotation = "" + if args.reference: + try: + with open(args.reference, encoding="utf-8") as handle: + reference = json.load(handle) + except (OSError, ValueError): + match = False + # A warning rather than a notice: a mismatch is the guard working, + # while an unreadable recording means it has nothing to work from, + # and a job that quietly measures nothing forever looks the same as + # one that measures. + annotation = ( + "::warning title=CodSpeed benchmarks skipped::No recording of the" + " machine the comparison base was measured on could be read at" + f" {args.reference}, so a measurement taken here cannot be shown" + " to be comparable with it." + ) + print(f"\nNo reference environment at {args.reference}.") + else: + moved = differences(current, reference) + if moved: + match = False + annotation = ( + "::notice title=CodSpeed benchmarks skipped::" + f"This runner ({current['cpu_brand']}, {current['digest']}) is" + " not the machine the comparison base was measured on, and an" + " upload from it would be published as a regression." + ) + # Digest the reference's own fields rather than reading back the + # digest stored beside them, so a recording written by an older + # field set is named by what it actually holds. + print( + f"\nThis machine is not the one the reference was taken on" + f" ({digest(reference)} -> {current['digest']}):" + ) + for name, was, now in moved: + if name == "cpu_flags": + was_set, now_set = set(was or ()), set(now or ()) + was = " ".join(sorted(was_set - now_set)) or "-" + now = " ".join(sorted(now_set - was_set)) or "-" + print(f" {name}: {was} -> {now}") + else: + print(f"\nThis machine matches the reference ({current['digest']}).") + + if args.record: + directory = os.path.dirname(args.record) + if directory: + os.makedirs(directory, exist_ok=True) + with open(args.record, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") + + if args.github_output: + # A skipped job is otherwise indistinguishable from one that never had + # benchmarks, and the reason is the one fact a reader needs. + if annotation: + print(annotation) + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write(f"match={'true' if match else 'false'}\n") + handle.write(f"digest={current['digest']}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main())