Update libc to 0.2.182 (#7247)

* Update to libc 0.2.182, refactor update script

* Fix typing

Cargo.lock

@@ -1721,9 +1721,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
 
 [[package]]
 name = "libc"
-version = "0.2.180"
+version = "0.2.182"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
+checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
 
 [[package]]
 name = "libffi"

Cargo.toml

@@ -182,7 +182,7 @@ insta = "1.46"
 itertools = "0.14.0"
 is-macro = "0.3.7"
 junction = "1.4.2"
-libc = "0.2.180"
+libc = "0.2.182"
 libffi = "5"
 log = "0.4.29"
 nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }

crates/vm/src/stdlib/posix.rs

@@ -61,14 +61,6 @@ pub mod module {
     use strum::IntoEnumIterator;
     use strum_macros::{EnumIter, EnumString};
 
-    #[cfg(any(target_os = "android", target_os = "linux"))]
-    #[pyattr]
-    use libc::{SCHED_DEADLINE, SCHED_NORMAL};
-
-    #[cfg(target_os = "freebsd")]
-    #[pyattr]
-    use libc::{MFD_HUGE_MASK, SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC};
-
     #[cfg(target_os = "linux")]
     #[pyattr]
     use libc::PIDFD_NONBLOCK;
@@ -80,51 +72,51 @@ pub mod module {
         PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, PRIO_DARWIN_THREAD,
     };
 
+    #[cfg(target_os = "freebsd")]
+    #[pyattr]
+    use libc::{SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC};
+
     #[cfg(any(target_os = "android", target_os = "linux"))]
     #[pyattr]
     use libc::{
         CLONE_FILES, CLONE_FS, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS,
         CLONE_NEWPID, CLONE_NEWUSER, CLONE_NEWUTS, CLONE_SIGHAND, CLONE_SYSVSEM, CLONE_THREAD,
-        CLONE_VM, EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, O_NOATIME, O_TMPFILE, P_PIDFD,
-        SCHED_BATCH, SCHED_IDLE, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE,
+        CLONE_VM, MFD_HUGE_SHIFT, O_NOATIME, O_TMPFILE, P_PIDFD, SCHED_BATCH, SCHED_DEADLINE,
+        SCHED_IDLE, SCHED_NORMAL, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE,
         SPLICE_F_NONBLOCK,
     };
 
-    #[cfg(any(target_os = "android", unix))]
-    #[pyattr]
-    use libc::{
-        F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER,
-        R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WNOHANG, WUNTRACED,
-        X_OK,
-    };
-
     #[cfg(any(target_os = "macos", target_os = "redox"))]
     #[pyattr]
     use libc::O_SYMLINK;
 
-    #[cfg(any(target_os = "android", target_os = "freebsd", target_os = "linux"))]
+    #[cfg(any(target_os = "android", target_os = "redox", unix))]
     #[pyattr]
-    use libc::{
-        MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGETLB, POSIX_FADV_DONTNEED, POSIX_FADV_NOREUSE,
-        POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL, POSIX_FADV_WILLNEED,
-    };
-
-    #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))]
-    #[pyattr]
-    use libc::{TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME, TFD_TIMER_CANCEL_ON_SET};
+    use libc::{O_NOFOLLOW, PRIO_PGRP, PRIO_PROCESS, PRIO_USER};
 
     #[cfg(any(target_os = "linux", target_os = "macos", target_os = "netbsd"))]
     #[pyattr]
     use libc::{XATTR_CREATE, XATTR_REPLACE};
 
-    #[cfg(any(
-        target_os = "android",
-        target_os = "dragonfly",
-        target_os = "linux",
-        target_os = "netbsd"
-    ))]
+    #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))]
     #[pyattr]
-    use libc::{GRND_NONBLOCK, GRND_RANDOM};
+    use libc::O_RSYNC;
+
+    #[cfg(any(target_os = "android", target_os = "freebsd", target_os = "linux"))]
+    #[pyattr]
+    use libc::{
+        MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGE_MASK, MFD_HUGETLB, POSIX_FADV_DONTNEED,
+        POSIX_FADV_NOREUSE, POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL,
+        POSIX_FADV_WILLNEED,
+    };
+
+    #[cfg(any(target_os = "android", target_os = "linux", target_os = "redox", unix))]
+    #[pyattr]
+    use libc::{RTLD_LAZY, RTLD_NOW, WNOHANG};
+
+    #[cfg(any(target_os = "android", target_os = "macos", target_os = "redox", unix))]
+    #[pyattr]
+    use libc::RTLD_GLOBAL;
 
     #[cfg(any(
         target_os = "android",
@@ -137,12 +129,64 @@ pub mod module {
 
     #[cfg(any(
         target_os = "android",
+        target_os = "freebsd",
         target_os = "linux",
-        target_os = "netbsd",
-        target_os = "openbsd"
+        target_os = "netbsd"
     ))]
     #[pyattr]
-    use libc::O_RSYNC;
+    use libc::{
+        EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME,
+        TFD_TIMER_CANCEL_ON_SET,
+    };
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "dragonfly",
+        target_os = "linux",
+        target_os = "netbsd"
+    ))]
+    #[pyattr]
+    use libc::{GRND_NONBLOCK, GRND_RANDOM};
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "linux",
+        target_os = "macos",
+        target_os = "redox",
+        unix
+    ))]
+    #[pyattr]
+    use libc::{F_OK, R_OK, W_OK, X_OK};
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "netbsd",
+        target_os = "redox",
+        unix
+    ))]
+    #[pyattr]
+    use libc::O_NONBLOCK;
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "linux",
+        target_os = "macos",
+        target_os = "netbsd"
+    ))]
+    #[pyattr]
+    use libc::O_DSYNC;
+
+    #[cfg(any(
+        target_os = "dragonfly",
+        target_os = "freebsd",
+        target_os = "linux",
+        target_os = "macos",
+        target_os = "netbsd"
+    ))]
+    #[pyattr]
+    use libc::SCHED_OTHER;
 
     #[cfg(any(
         target_os = "android",
@@ -164,6 +208,38 @@ pub mod module {
     #[pyattr]
     use libc::O_DIRECT;
 
+    #[cfg(any(
+        target_os = "dragonfly",
+        target_os = "freebsd",
+        target_os = "macos",
+        target_os = "netbsd",
+        target_os = "redox"
+    ))]
+    #[pyattr]
+    use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK};
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "linux",
+        target_os = "macos",
+        target_os = "netbsd",
+        target_os = "redox",
+        unix
+    ))]
+    #[pyattr]
+    use libc::RTLD_LOCAL;
+
+    #[cfg(any(
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "linux",
+        target_os = "netbsd",
+        target_os = "redox",
+        unix
+    ))]
+    #[pyattr]
+    use libc::WUNTRACED;
+
     #[cfg(any(
         target_os = "android",
         target_os = "dragonfly",
@@ -173,54 +249,35 @@ pub mod module {
         target_os = "netbsd"
     ))]
     #[pyattr]
-    use libc::RTLD_NOLOAD;
+    use libc::{
+        CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, O_SYNC, P_ALL,
+        P_PGID, P_PID, RTLD_NOLOAD, SCHED_FIFO, SCHED_RR,
+    };
 
     #[cfg(any(
         target_os = "android",
+        target_os = "dragonfly",
         target_os = "freebsd",
-        target_os = "linux",
         target_os = "macos",
         target_os = "netbsd",
-        target_os = "openbsd"
+        target_os = "redox",
+        unix
     ))]
     #[pyattr]
-    use libc::O_DSYNC;
+    use libc::O_DIRECTORY;
 
     #[cfg(any(
+        target_os = "android",
         target_os = "dragonfly",
         target_os = "freebsd",
         target_os = "linux",
         target_os = "macos",
         target_os = "netbsd",
-        target_os = "openbsd"
-    ))]
-    #[pyattr]
-    use libc::SCHED_OTHER;
-
-    #[cfg(any(
-        target_os = "dragonfly",
-        target_os = "freebsd",
-        target_os = "macos",
-        target_os = "netbsd",
-        target_os = "openbsd",
         target_os = "redox"
     ))]
     #[pyattr]
-    use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK};
-
-    #[cfg(any(
-        target_os = "android",
-        target_os = "dragonfly",
-        target_os = "freebsd",
-        target_os = "linux",
-        target_os = "macos",
-        target_os = "netbsd",
-        target_os = "openbsd"
-    ))]
-    #[pyattr]
     use libc::{
-        CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, F_LOCK,
-        F_TEST, F_TLOCK, F_ULOCK, O_SYNC, P_ALL, P_PGID, P_PID, SCHED_FIFO, SCHED_RR,
+        F_LOCK, F_TEST, F_TLOCK, F_ULOCK, O_ASYNC, O_NDELAY, O_NOCTTY, WEXITED, WNOWAIT, WSTOPPED,
     };
 
     #[cfg(any(
@@ -230,11 +287,11 @@ pub mod module {
         target_os = "linux",
         target_os = "macos",
         target_os = "netbsd",
-        target_os = "openbsd",
-        target_os = "redox"
+        target_os = "redox",
+        unix
     ))]
     #[pyattr]
-    use libc::{O_ASYNC, O_NDELAY, O_NOCTTY, WEXITED, WNOWAIT, WSTOPPED};
+    use libc::{O_CLOEXEC, WCONTINUED};
 
     #[pyattr]
     const EX_OK: i8 = exitcode::OK as i8;

scripts/libc_posix.py

@@ -1,19 +1,24 @@
 #!/usr/bin/env python
 import collections
+import dataclasses
+import pathlib
 import re
+import subprocess
 import urllib.request
-from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
-    from collections.abc import Iterator
+import tomllib
 
-CONSTS_PAT = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b")
-OS_CONSTS_PAT = re.compile(
-    r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)"
-)  # TODO: Exclude matches if they have `(` after (those are functions)
+CPYTHON_VERSION = "3.14"
+
+CONSTS_PATTERN = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b")
+
+CARGO_TOML_FILE = pathlib.Path(__file__).parents[1] / "Cargo.toml"
+CARGO_TOML = tomllib.loads(CARGO_TOML_FILE.read_text())
+LIBC_DATA = CARGO_TOML["workspace"]["dependencies"]["libc"]
 
 
-LIBC_VERSION = "0.2.180"
+LIBC_VERSION = LIBC_DATA["version"] if isinstance(LIBC_DATA, dict) else LIBC_DATA
+BASE_URL = f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}"
 
 EXCLUDE = frozenset(
     {
@@ -42,89 +47,201 @@ EXCLUDE = frozenset(
     }
 )
 
-EXTRAS = {
-    frozenset({"macos"}): {"COPYFILE_DATA"},
-}
-RENAMES = {"COPYFILE_DATA": "_COPYFILE_DATA"}
+
+def rustfmt(code: str) -> str:
+    return subprocess.check_output(["rustfmt", "--emit=stdout"], input=code, text=True)
 
 
-def build_url(fname: str) -> str:
-    return f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}/libc-test/semver/{fname}.txt"
+@dataclasses.dataclass(eq=True, frozen=True, slots=True)
+class Cfg:
+    inner: str
+
+    def __str__(self) -> str:
+        return self.inner
+
+    def __lt__(self, other) -> bool:
+        si, oi = map(str, (self.inner, other.inner))
+
+        # Smaller length cfgs are smaller, regardless of value.
+        return (len(si), si) < (len(oi), oi)
 
 
-TARGET_OS = {
-    "android": "android",
-    "dragonfly": "dragonfly",
-    "freebsd": "freebsd",
-    "linux": "linux",
-    "macos": "apple",
-    "netbsd": "netbsd",
-    "openbsd": "openbsd",
-    "redox": "redox",
-    # solaris?
-    "unix": "unix",
-}
+@dataclasses.dataclass(frozen=True, kw_only=True, slots=True)
+class Target:
+    cfgs: set[Cfg]
+    sources: set[str] = dataclasses.field(default_factory=set)
+    extras: set[str] = dataclasses.field(default_factory=set)
 
 
-def get_consts(url: str, pattern: re.Pattern = CONSTS_PAT) -> frozenset[str]:
-    with urllib.request.urlopen(url) as f:
-        resp = f.read().decode()
-
-    return frozenset(pattern.findall(resp)) - EXCLUDE
+TARGETS = (
+    Target(
+        cfgs={Cfg('target_os = "android"')},
+        sources={
+            f"{BASE_URL}/src/unix/linux_like/android/mod.rs",
+            f"{BASE_URL}/libc-test/semver/android.txt",
+        },
+    ),
+    Target(
+        cfgs={Cfg('target_os = "dragonfly"')},
+        sources={
+            f"{BASE_URL}/src/unix/bsd/freebsdlike/dragonfly/mod.rs",
+            f"{BASE_URL}/libc-test/semver/dragonfly.txt",
+        },
+    ),
+    Target(
+        cfgs={Cfg('target_os = "freebsd"')},
+        sources={
+            f"{BASE_URL}/src/unix/bsd/freebsdlike/freebsd/mod.rs",
+            f"{BASE_URL}/libc-test/semver/freebsd.txt",
+        },
+    ),
+    Target(
+        cfgs={Cfg('target_os = "linux"')},
+        sources={
+            f"{BASE_URL}/src/unix/linux_like/mod.rs",
+            f"{BASE_URL}/src/unix/linux_like/linux_l4re_shared.rs",
+            f"{BASE_URL}/libc-test/semver/linux.txt",
+        },
+    ),
+    Target(
+        cfgs={Cfg('target_os = "macos"')},
+        sources={
+            f"{BASE_URL}/src/unix/bsd/apple/mod.rs",
+            f"{BASE_URL}/libc-test/semver/apple.txt",
+        },
+        extras={"COPYFILE_DATA as _COPYFILE_DATA"},
+    ),
+    Target(
+        cfgs={Cfg('target_os = "netbsd"')},
+        sources={
+            f"{BASE_URL}/src/unix/bsd/netbsdlike/netbsd/mod.rs",
+            f"{BASE_URL}/libc-test/semver/netbsd.txt",
+        },
+    ),
+    Target(
+        cfgs={Cfg('target_os = "redox"')},
+        sources={
+            f"{BASE_URL}/src/unix/redox/mod.rs",
+            f"{BASE_URL}/libc-test/semver/redox.txt",
+        },
+    ),
+    Target(cfgs={Cfg("unix")}, sources={f"{BASE_URL}/libc-test/semver/unix.txt"}),
+)
 
 
-def format_groups(groups: dict) -> "Iterator[tuple[str, str]]":
-    # sort by length, then alphabet. so we will have a consistent output
-    for targets, consts in sorted(
-        groups.items(), key=lambda t: (len(t[0]), sorted(t[0]))
-    ):
-        cond = ", ".join(
-            f'target_os = "{target_os}"' if target_os != "unix" else target_os
-            for target_os in sorted(targets)
-        )
-        if len(targets) > 1:
-            cond = f"any({cond})"
-        cfg = f"#[cfg({cond})]"
+def extract_consts(
+    contents: str,
+    *,
+    pattern: re.Pattern = CONSTS_PATTERN,
+    exclude: frozenset[str] = EXCLUDE,
+) -> frozenset[str]:
+    """
+    Extract all words that are comprised from only uppercase letters + underscores.
 
-        imports = ", ".join(
-            const if const not in RENAMES else f"{const} as {RENAMES[const]}"
-            for const in sorted(consts)
-        )
-        use = f"use libc::{{{imports}}};"
-        yield cfg, use
+    Parameters
+    ----------
+    contents : str
+        Contents to extract the constants from.
+    pattern : re.Pattern, Optional
+        RE compiled pattern for extracting the consts.
+    exclude : frozenset[str], Optional
+        Items to exclude from the returned value.
+
+    Returns
+    -------
+    frozenset[str]
+        All constant names.
+    """
+    result = frozenset(pattern.findall(contents))
+    return result - exclude
+
+
+def consts_from_url(
+    url: str, *, pattern: re.Pattern = CONSTS_PATTERN, exclude: frozenset[str] = EXCLUDE
+) -> frozenset[str]:
+    """
+    Extract all consts from the contents found at the given URL.
+
+    Parameters
+    ----------
+    url : str
+        URL to fetch the contents from.
+    pattern : re.Pattern, Optional
+        RE compiled pattern for extracting the consts.
+    exclude : frozenset[str], Optional
+        Items to exclude from the returned value.
+
+    Returns
+    -------
+    frozenset[str]
+        All constant names at the URL.
+    """
+    try:
+        with urllib.request.urlopen(url) as f:
+            contents = f.read().decode()
+    except urllib.error.HTTPError as err:
+        err.add_note(url)
+        raise
+
+    return extract_consts(contents, pattern=pattern, exclude=exclude)
 
 
 def main():
-    wanted_consts = get_consts(
-        "https://docs.python.org/3.14/library/os.html",  # Should we read from https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Modules/posixmodule.c#L17023 instead?
-        pattern=OS_CONSTS_PAT,
+    # Step 1: Get all OS contants that we do want from upstream
+    wanted_consts = consts_from_url(
+        f"https://docs.python.org/{CPYTHON_VERSION}/library/os.html",
+        # TODO: Exclude matches if they have `(` after (those are functions)
+        pattern=re.compile(r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)"),
     )
-    available = {
-        target_os: get_consts(build_url(fname))
-        for target_os, fname in TARGET_OS.items()
-    }
 
-    group_consts = collections.defaultdict(set)
-    for const in wanted_consts:
-        target_oses = frozenset(
-            target_os for target_os, consts in available.items() if const in consts
+    # Step 2: build dict of what consts are available per cfg. `cfg -> {consts}`
+    available = collections.defaultdict(set)
+    for target in TARGETS:
+        consts = set()
+        for source in target.sources:
+            consts |= consts_from_url(source)
+
+        for cfg in target.cfgs:
+            available[cfg] |= consts
+
+    # Step 3: Keep only the "wanted" consts. Build a groupped mapping of `{cfgs} -> {consts}'
+    groups = collections.defaultdict(set)
+    available_items = available.items()
+    for wanted_const in wanted_consts:
+        cfgs = frozenset(
+            cfg for cfg, consts in available_items if wanted_const in consts
         )
-        if not target_oses:
+        if not cfgs:
+            # We have no cfgs for a wanted const :/
             continue
 
-        group_consts[target_oses].add(const)
-    group_consts = {grp: v | EXTRAS.get(grp, set()) for grp, v in group_consts.items()}
+        groups[cfgs].add(wanted_const)
 
-    code = "\n\n".join(
-        f"""
-{cfg}
+    # Step 4: Build output
+    output = ""
+    for cfgs, consts in sorted(groups.items(), key=lambda t: (len(t[0]), sorted(t[0]))):
+        target = next((target for target in TARGETS if target.cfgs == cfgs), None)
+        if target:
+            # If we found an exact target. Add its "extras" as-is
+            consts |= target.extras
+
+        cfgs_inner = ",".join(sorted(map(str, cfgs)))
+
+        if len(cfgs) >= 2:
+            cfgs_rust = f"#[cfg(any({cfgs_inner}))]"
+        else:
+            cfgs_rust = f"#[cfg({cfgs_inner})]"
+
+        imports = ",".join(consts)
+        entry = f"""
+{cfgs_rust}
 #[pyattr]
-{use}
+use libc::{{{imports}}};
 """.strip()
-        for cfg, use in format_groups(group_consts)
-    )
 
-    print(code)
+        output += f"{entry}\n\n"
+
+    print(rustfmt(output))
 
 
 if __name__ == "__main__":