From 0e353134844fd50628f3730b9f704c3de1710e20 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Mon, 26 Oct 2026 15:09:33 -0700 Subject: [PATCH] gh-160689 : Link only the bootstrap modules into the build helper programs (#160537) The build-helper programs (`Programs/_freeze_module` and `_bootstrap_python`) don't need to link extension modules that they don't need. This PR makes sure the helper programs get only the modules they need to do their job, regardless of what the build configuration sets for the final built runtime and interpreter. This follows the same pattern as what `PCbuild/_freeze_module.vcxproj` does with `PC/config_minimal.c`, and lets "extra" builtin modules (from `Setup.local` or `MODULE_BUILDTYPE=static`) depend on frozen modules or `getpath`. --- .gitignore | 1 + Makefile.pre.in | 69 ++++++++++--- ...-10-25-15-30-00.gh-issue-160689.bootsp.rst | 5 + Modules/makesetup | 96 +++++++++++++++---- Tools/c-analyzer/cpython/ignored.tsv | 1 + configure | 4 +- configure.ac | 4 +- 7 files changed, 143 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst diff --git a/.gitignore b/.gitignore index 0212701e9a3..d82d89cd2df 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ Modules/Setup.config Modules/Setup.local Modules/Setup.stdlib Modules/config.c +Modules/config_bootstrap.c Modules/ld_so_aix Programs/_freeze_module Programs/_testembed diff --git a/Makefile.pre.in b/Makefile.pre.in index c2d1a75292e..b0680f595a9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -25,6 +25,13 @@ MODSHARED_NAMES= _MODSHARED_NAMES_ MODDISABLED_NAMES= _MODDISABLED_NAMES_ MODOBJS= _MODOBJS_ MODLIBS= _MODLIBS_ +# Objects and libraries of the bootstrap modules (Modules/Setup.bootstrap) +# only. These are the only static modules linked into the build helper +# programs (Programs/_freeze_module, _bootstrap_python), so that other +# builtin modules (Modules/Setup.local, MODULE_BUILDTYPE=static) may depend +# on products of those helpers (frozen modules, getpath). +BOOTSTRAP_MODOBJS= _BOOTSTRAP_MODOBJS_ +BOOTSTRAP_MODLIBS= _BOOTSTRAP_MODLIBS_ # === Variables set by configure VERSION= @VERSION@ @@ -359,11 +366,14 @@ COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "C ########################################################################## # Modules -MODULE_OBJS= \ - Modules/config.o \ +MODULE_OBJS_COMMON= \ Modules/main.o \ Modules/gcmodule.o +MODULE_OBJS= \ + Modules/config.o \ + $(MODULE_OBJS_COMMON) + IO_H= Modules/_io/_iomodule.h IO_OBJS= \ @@ -614,6 +624,19 @@ LIBRARY_OBJS= \ Modules/getpath.o \ Python/frozen.o +# objects that get linked into the build helper programs +# (Programs/_freeze_module, _bootstrap_python): the core, plus the bootstrap +# modules only (see BOOTSTRAP_MODOBJS). Does not include frozen modules nor +# getpath, which the helpers are used to build. +BOOTSTRAP_LIBRARY_OBJS= \ + Modules/getbuildinfo_bootstrap.o \ + $(PARSER_OBJS) \ + $(OBJECT_OBJS) \ + $(PYTHON_OBJS) \ + Modules/config_bootstrap.o \ + $(MODULE_OBJS_COMMON) \ + $(BOOTSTRAP_MODOBJS) + LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@ ########################################################################## @@ -1613,7 +1636,7 @@ Modules/Setup.bootstrap: $(srcdir)/Modules/Setup.bootstrap.in config.status Modules/Setup.stdlib: $(srcdir)/Modules/Setup.stdlib.in config.status ./config.status $@ -Makefile Modules/config.c: Makefile.pre \ +Makefile Modules/config.c Modules/config_bootstrap.c: Makefile.pre \ $(srcdir)/Modules/config.c.in \ $(MAKESETUP) \ $(srcdir)/Modules/Setup \ @@ -1624,9 +1647,9 @@ Makefile Modules/config.c: Makefile.pre \ -s Modules \ Modules/Setup.local \ Modules/Setup.stdlib \ - Modules/Setup.bootstrap \ + -b Modules/Setup.bootstrap \ $(srcdir)/Modules/Setup - @mv config.c Modules + @mv config.c config_bootstrap.c Modules @echo "The Makefile was updated, you may need to re-run make." .PHONY: regen-test-frozenmain @@ -1661,9 +1684,9 @@ BOOTSTRAP_HEADERS = \ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) -_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local - $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ - Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) +_bootstrap_python: $(BOOTSTRAP_LIBRARY_OBJS) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local + $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(BOOTSTRAP_LIBRARY_OBJS) \ + Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS) # Dummy pybuilddir.txt is needed for _bootstrap_python to be runnable @echo "none" > ./pybuilddir.txt @@ -1681,6 +1704,9 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Standard build process: # 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*(). +# Only the bootstrap modules (Modules/Setup.bootstrap) are part of these +# objects (see BOOTSTRAP_LIBRARY_OBJS), so other builtin modules may +# depend on the output of the following steps. # 2) build Programs/_freeze_module binary. # 3) create frozen module headers for importlib and getpath. # 4) build _bootstrap_python binary. @@ -1753,8 +1779,8 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile -Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS) + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS) $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS) # We manually freeze getpath.py rather than through freeze_modules Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) @@ -1907,11 +1933,24 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(DTRACE_OBJS) \ $(srcdir)/Modules/getbuildinfo.c - $(CC) -c $(PY_CORE_CFLAGS) \ + $(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \ + -o $@ $(srcdir)/Modules/getbuildinfo.c + +# Separate object for the build helper programs: it must not depend on +# $(MODOBJS), which may in turn depend on the helpers' output. +Modules/getbuildinfo_bootstrap.o: $(PARSER_OBJS) \ + $(OBJECT_OBJS) \ + $(PYTHON_OBJS) \ + $(MODULE_OBJS_COMMON) \ + $(BOOTSTRAP_MODOBJS) \ + $(srcdir)/Modules/getbuildinfo.c + $(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \ + -o $@ $(srcdir)/Modules/getbuildinfo.c + +GETBUILDINFO_CFLAGS= \ -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ - -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ - -o $@ $(srcdir)/Modules/getbuildinfo.c + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \ @@ -2279,7 +2318,7 @@ regen-slots: Python/slots.toml $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \ --generate-all -$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS) +$(LIBRARY_OBJS) $(MODOBJS) Modules/config_bootstrap.o Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS) ###################################################################### @@ -3365,7 +3404,7 @@ clean: clean-profile clean-jit-stencils clobber: clean -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ tags TAGS \ - config.cache config.log pyconfig.h Modules/config.c + config.cache config.log pyconfig.h Modules/config.c Modules/config_bootstrap.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -rf Platforms/Apple/iOS/Frameworks diff --git a/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst b/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst new file mode 100644 index 00000000000..e83c9708142 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst @@ -0,0 +1,5 @@ +The build helper programs ``Programs/_freeze_module`` and +``_bootstrap_python`` are now linked only with the modules of +:file:`Modules/Setup.bootstrap`, instead of with every static module. Builtin +modules added via :file:`Modules/Setup.local` or ``MODULE_BUILDTYPE=static`` +can now depend on frozen modules or on ``getpath``. diff --git a/Modules/makesetup b/Modules/makesetup index 8b28946608b..932c0952e47 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -4,17 +4,22 @@ set -e # Convert templates into Makefile and config.c, based on the module # definitions found in the file Setup. # -# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-n [Setup] ...] +# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-b Setup] ... +# [-n [Setup] ...] # # Options: # -s directory: alternative source directory (default .) # -l directory: library source directory (default derived from $0) # -c file: alternative config.c template (default $libdir/config.c.in) -# -c -: don't write config.c +# -c -: don't write config.c (nor config_bootstrap.c) # -m file: alternative Makefile template (default ./Makefile.pre) # -m -: don't write Makefile # # Remaining arguments are one or more Setup files (default ./Setup). +# The Setup file following a -b option holds the *bootstrap* modules: +# the static modules that the build helper programs (_freeze_module, +# _bootstrap_python) are linked with. They are also regular modules of +# the final build. # Setup files after a -n option are used for their variables, modules # and libraries but not for their .o files. # @@ -28,6 +33,9 @@ set -e # + insert 'extern PyObject* PyInit_(void);' before MARKER 1 # + insert '{"", PyInit_},' before MARKER 2 # +# Copying config.c.in to config_bootstrap.c (only with a -b Setup file): +# - same as config.c, but only for the modules of the -b Setup file +# # Copying Makefile.pre to Makefile: # - insert an identifying comment at the start # - replace _MODBUILT_NAMES_ by the list of *static* and *shared* modules @@ -37,6 +45,8 @@ set -e # - replace _MODOBJS_ by the list of objects from Setup (except for # Setup files after a -n option) # - replace _MODLIBS_ by the list of libraries from Setup +# - replace _BOOTSTRAP_MODOBJS_ and _BOOTSTRAP_MODLIBS_ by the objects and +# libraries of the -b Setup file # - for each object file mentioned in Setup, append a rule # '.o: .c; ' to the end of the Makefile # - for each module mentioned in Setup, append a rule @@ -47,13 +57,14 @@ set -e # Loop over command line options usage=' usage: makesetup [-s srcdir] [-l libdir] [-c config.c.in] [-m Makefile.pre] - [Setup] ... [-n [Setup] ...]' + [Setup] ... [-b Setup] ... [-n [Setup] ...]' srcdir='.' libdir='' config='' makepre='' noobjects='' doconfig=yes +bootstrap= while : do case $1 in @@ -93,7 +104,8 @@ do echo '' # Add a linebreak so we don't choke on files missing EOL. case $i in -n) echo '*noobjects*';; - *) echo '*doconfig*'; cat "$i";; + -b) echo '*bootstrap*';; + *) echo '*doconfig*'; cat "$i"; echo ''; echo '*nobootstrap*';; esac done | sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | @@ -114,6 +126,10 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | LIBS= LOCALLIBS= BASELIBS= + BOOTMODS= + BOOTOBJS= + BOOTLIBS= + havebootstrap= while read line do # to handle backslashes for sh's that don't automatically @@ -136,6 +152,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | noobjects=yes; continue;; '*doconfig*') doconfig=yes; continue;; + '*bootstrap*') bootstrap=yes; havebootstrap=yes; continue;; + '*nobootstrap*') bootstrap=; continue;; '*static*') doconfig=yes; continue;; '*noconfig*') doconfig=no; continue;; '*shared*') doconfig=no; continue;; @@ -204,6 +222,18 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | # Detected multiple rules for a module, first rule wins. This # allows users to disable modules in Setup.local. echo 1>&2 "maksetup: '$mod' was handled by previous rule." + case $bootstrap in + yes) + # This module is in the bootstrap Setup file, but an + # earlier Setup file (e.g. Setup.local) already defined + # it, and that definition won. Give the build helpers + # that earlier definition, so they use the same module + # as the final interpreter. + eval "BOOTMODS=\"\$BOOTMODS \${STATICMOD_$mod}\"" + eval "BOOTOBJS=\"\$BOOTOBJS \${STATICOBJS_$mod}\"" + eval "BOOTLIBS=\"\$BOOTLIBS \${STATICLIBS_$mod}\"" + ;; + esac continue 2;; esac CONFIGURED="$CONFIGURED,${mod}," @@ -213,6 +243,12 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | LIBS="$LIBS $libs" MODS="$MODS $mods" BUILT="$BUILT $mods" + case $bootstrap in + yes) + BOOTLIBS="$BOOTLIBS $libs" + BOOTMODS="$BOOTMODS $mods" + ;; + esac ;; no) BUILT="$BUILT $mods" @@ -265,7 +301,21 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | echo "$rule" >>$rulesf done case $doconfig in - yes) OBJS="$OBJS $objs";; + yes) + OBJS="$OBJS $objs" + case $bootstrap in + yes) BOOTOBJS="$BOOTOBJS $objs";; + esac + # Record each static module's objects and libs by name, so + # the bootstrap handling above can find a definition that + # won over a later line in the bootstrap Setup file. + for mod in $mods + do + eval "STATICMOD_$mod='$mod'" + eval "STATICOBJS_$mod='$objs'" + eval "STATICLIBS_$mod='$libs'" + done + ;; esac for mod in $mods do @@ -295,24 +345,31 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | DEFS="BASEMODLIBS=$BASELIBS$NL$DEFS" DEFS="LOCALMODLIBS=$LOCALLIBS$NL$DEFS" - EXTDECLS= - INITBITS= - for mod in $MODS - do - EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL" - INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL" - done - - - case $config in - -) ;; - *) sed -e " + # Generate a config.c from the template for the given modules + genconfig() { + out=$1; shift + EXTDECLS= + INITBITS= + for mod + do + EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL" + INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL" + done + sed -e " 1i$NL/* Generated automatically from $config by makesetup. */ /MARKER 1/i$NL$EXTDECLS /MARKER 2/i$NL$INITBITS - " $config >config.c + " $config >$out + } + + case $config in + -) ;; + *) genconfig config.c $MODS + case $havebootstrap in + yes) genconfig config_bootstrap.c $BOOTMODS;; + esac ;; esac @@ -331,6 +388,9 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | echo "s%_MODBUILT_NAMES_%$BUILT%" >>$sedf echo "s%_MODSHARED_NAMES_%$BUILT_SHARED%" >>$sedf echo "s%_MODDISABLED_NAMES_%$DISABLED%" >>$sedf + # _BOOTSTRAP_* first: _MODOBJS_ is a substring of _BOOTSTRAP_MODOBJS_ + echo "s%_BOOTSTRAP_MODOBJS_%$BOOTOBJS%" >>$sedf + echo "s%_BOOTSTRAP_MODLIBS_%$BOOTLIBS%" >>$sedf echo "s%_MODOBJS_%$OBJS%" >>$sedf echo "s%_MODLIBS_%$LIBS%" >>$sedf echo "/Definitions added by makesetup/r $sedr" >>$sedf diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index db9f74fdbc5..62f454b6355 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -314,6 +314,7 @@ Modules/cmathmodule.c - sinh_special_values - Modules/cmathmodule.c - sqrt_special_values - Modules/cmathmodule.c - tanh_special_values - Modules/config.c - _PyImport_Inittab - +Modules/config_bootstrap.c - _PyImport_Inittab - Modules/faulthandler.c - faulthandler_handlers - Modules/getnameinfo.c - gni_afdl - Modules/hmacmodule.c - py_hmac_static_hinfo - diff --git a/configure b/configure index 1d745f45ca9..a11cd0c27bc 100755 --- a/configure +++ b/configure @@ -40684,12 +40684,12 @@ fi printf "%s\n" "$as_me: creating Makefile" >&6;} $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \ -s Modules \ - Modules/Setup.local Modules/Setup.stdlib Modules/Setup.bootstrap $srcdir/Modules/Setup + Modules/Setup.local Modules/Setup.stdlib -b Modules/Setup.bootstrap $srcdir/Modules/Setup if test $? -ne 0; then as_fn_error $? "makesetup failed" "$LINENO" 5 fi -mv config.c Modules +mv config.c config_bootstrap.c Modules if test -z "$PKG_CONFIG"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: pkg-config is missing. Some dependencies may not be detected correctly." >&5 diff --git a/configure.ac b/configure.ac index df4a7494ecd..640919bd07b 100644 --- a/configure.ac +++ b/configure.ac @@ -9070,12 +9070,12 @@ fi AC_MSG_NOTICE([creating Makefile]) $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \ -s Modules \ - Modules/Setup.local Modules/Setup.stdlib Modules/Setup.bootstrap $srcdir/Modules/Setup + Modules/Setup.local Modules/Setup.stdlib -b Modules/Setup.bootstrap $srcdir/Modules/Setup if test $? -ne 0; then AC_MSG_ERROR([makesetup failed]) fi -mv config.c Modules +mv config.c config_bootstrap.c Modules if test -z "$PKG_CONFIG"; then AC_MSG_WARN([pkg-config is missing. Some dependencies may not be detected correctly.])