Add ARM optimized-routines for aarch64 scalar math and string functions

New contrib optimized-routines (upstream ARM-software/optimized-routines,
no fork needed - built as-is): on aarch64, scalar exp, exp2, log, log10,
atan, atan2 replace the llvm-libc generic implementations (excluded from
libllvmlibc there), and the AdvSIMD/MTE string routines - memcpy,
memmove, memcmp, memchr, strlen, strcmp, strchr, strnlen - replace the
mix of musl generic C, musl's vendored 2020 AOR asm, and llvm-libc weak
definitions. Both archives are linked with --whole-archive, so the
definitions win the link structurally.

Measured on Graviton4 (Neoverse-V2) against the previous static-binary
state: exp 2.1x, exp2 2.2x, atan 1.7x, atan2 1.6x, memcpy +26-37%,
memmove ~2x on small sizes, memcmp +50-80%, strlen 2.9x, memchr 3x.
This closes the gap to glibc, whose aarch64 libm and ifunc-selected
string routines derive from the same upstream. The SVE string variants
are deliberately not used: on 128-bit-SVE cores they measure up to 3x
slower than the NEON/MTE ones. AOR math scalars are 0.5-2.5 ULP instead
of llvm-libc's correct rounding, matching glibc's accuracy class.

Motivated by the h3ToGeoBoundary regression in the perf comparison:
https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=109239

Verified under qemu-aarch64 with a static test binary: exp/exp2/log/
log10/atan/atan2 exact on reference points; strlen/memcmp/memchr/strcmp
correct incl. the MTE variants. x86_64 builds are unaffected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitmodules

@@ -423,3 +423,6 @@
 [submodule "contrib/MeCab"]
 	path = contrib/MeCab
 	url = https://github.com/taku910/mecab
+[submodule "contrib/optimized-routines"]
+	path = contrib/optimized-routines
+	url = https://github.com/ARM-software/optimized-routines

cmake/linux/default_libs.cmake

@@ -20,6 +20,14 @@ if (ENABLE_LLVM_LIBC_MATH)
     target_link_libraries(global-libs INTERFACE libllvmlibc)
     set (DEFAULT_LIBS "${DEFAULT_LIBS} -llibllvmlibc")
 
+    if (ARCH_AARCH64)
+        # Scalar exp/exp2/log/log10/atan/atan2 from ARM's optimized-routines;
+        # excluded from libllvmlibc there (see contrib/optimized-routines-cmake).
+        link_directories("${CMAKE_BINARY_DIR}/contrib/optimized-routines-cmake")
+        target_link_libraries(global-libs INTERFACE aor)
+        set (DEFAULT_LIBS "${DEFAULT_LIBS} -laor")
+    endif()
+
     if (NOT SANITIZE)
         # Force every llvm-libc member into the link ahead of all objects and archives
         # (linker flags precede them on the link line; -L placement does not matter to ld).
@@ -33,6 +41,9 @@ if (ENABLE_LLVM_LIBC_MATH)
         # Skipped under sanitizers, where the interceptors must wrap the libc mem functions
         # instead (same reason the -u forcing was skipped before).
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--whole-archive -llibllvmlibc -Wl,--no-whole-archive")
+        if (ARCH_AARCH64)
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--whole-archive -laor -Wl,--no-whole-archive")
+        endif()
         # Redundant with --whole-archive above, but kept as a backstop for the mem functions
         # in case the whole-archive link is ever weakened or repositioned.
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-u,memcpy -Wl,-u,memmove -Wl,-u,memset -Wl,-u,memcmp")

contrib/CMakeLists.txt

@@ -60,6 +60,7 @@ add_contrib (google-protobuf-cmake google-protobuf)
 add_contrib (grpc-cmake grpc)
 add_contrib (compiler-rt-cmake llvm-project)
 add_contrib (libllvmlibc-cmake llvm-project)
+add_contrib (optimized-routines-cmake optimized-routines)
 add_contrib (llvm-project-cmake llvm-project)
 
 enable_dummy_launchers_if_needed()

contrib/corrosion-cmake/CMakeLists.txt

@@ -386,7 +386,7 @@ function(clickhouse_config_crate_flags target_name)
         # caller of this function.
         set(_localize_ref_libs "")
         set(_localize_ref_deps "")
-        foreach (_ref_target libllvmlibc clang_rt_builtins)
+        foreach (_ref_target libllvmlibc aor clang_rt_builtins)
             if (TARGET ${_ref_target})
                 list(APPEND _localize_ref_libs "$<TARGET_FILE:${_ref_target}>")
                 list(APPEND _localize_ref_deps ${_ref_target})

contrib/libllvmlibc-cmake/CMakeLists.txt

@@ -68,6 +68,22 @@ if (NOT SANITIZE)
     list (APPEND SRCS "${CMAKE_CURRENT_SOURCE_DIR}/string/memmem.cpp")
 endif ()
 
+# On aarch64 these functions come from ARM's optimized-routines instead
+# (contrib/optimized-routines-cmake), which beats llvm-libc's generic
+# implementations on Neoverse cores (exp 2.1x, exp2 2.2x, atan 1.7x,
+# atan2 1.6x, plus log/log10; measured on Graviton4). Both archives are
+# linked with --whole-archive, so the sources must not overlap.
+if (ARCH_AARCH64)
+    list(REMOVE_ITEM SRCS
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/exp.cpp"
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/exp2.cpp"
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/log.cpp"
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/log10.cpp"
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/atan.cpp"
+        "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/atan2.cpp"
+    )
+endif()
+
 set(LIBC_NAMESPACE "__llvm_libc")
 set(LLVMLIBC_DEFS_COMMON
     LIBC_MATH_HAS_NO_ERRNO=1

contrib/optimized-routines-cmake/CMakeLists.txt

@@ -0,0 +1,95 @@
+# Selected routines from ARM's optimized-routines (AOR), the upstream that
+# glibc's aarch64 libm and string routines derive from. Built only for
+# aarch64, and only where AOR measurably beats what the static binary
+# otherwise resolves (benchmarked on Graviton4 / Neoverse-V2, see the perf
+# comparison of PR 109239):
+#
+#   math (vs llvm-libc generic):   exp 2.1x, exp2 2.2x, log, log10,
+#                                  atan 1.7x, atan2 1.6x
+#   string (vs musl generic C, musl's vendored 2020 AOR asm, and llvm-libc):
+#                                  memcpy +26-37%, memmove ~2x small sizes,
+#                                  memcmp +50-80%, strlen 2.9x, memchr 3x
+#
+# The corresponding math sources are excluded from libllvmlibc on aarch64
+# (see contrib/libllvmlibc-cmake), and this archive is linked with
+# --whole-archive next to it (see cmake/linux/default_libs.cmake), so these
+# definitions win the link the same way. The string asm is strong and wins
+# over llvm-libc's weak memmove/memcmp; musl's own copies in libmusl.a are
+# ordinary archive members and are simply never extracted.
+#
+# Accuracy: AOR math scalars are 0.5-2.5 ULP (the file names encode the
+# bound, e.g. atan2_2u5.c = 2.5 ULP) instead of llvm-libc's correct
+# rounding; this matches the accuracy class of glibc's aarch64 libm.
+#
+# The *-mte variants are used for strlen/memchr/strchr: they never read
+# beyond the granule containing the terminator, are valid on any core, and
+# measured fastest on Neoverse-V2. The SVE variants are deliberately not
+# used: on 128-bit-SVE cores (Graviton4) they are up to 3x slower than the
+# NEON/MTE ones.
+#
+# On x86_64 llvm-libc's implementations (built with AVX2/FMA) are
+# competitive, so this library is not built there.
+
+if (NOT ARCH_AARCH64)
+    return()
+endif()
+
+set(AOR_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/optimized-routines")
+
+set(SRCS
+    "${AOR_SOURCE_DIR}/math/exp.c"
+    "${AOR_SOURCE_DIR}/math/exp_data.c"
+    "${AOR_SOURCE_DIR}/math/exp2.c"
+    "${AOR_SOURCE_DIR}/math/log.c"
+    "${AOR_SOURCE_DIR}/math/log_data.c"
+    "${AOR_SOURCE_DIR}/math/math_err.c"
+    "${AOR_SOURCE_DIR}/math/aarch64/experimental/atan_2u5.c"
+    "${AOR_SOURCE_DIR}/math/aarch64/experimental/atan2_2u5.c"
+    "${AOR_SOURCE_DIR}/math/aarch64/experimental/atan_data.c"
+    "${AOR_SOURCE_DIR}/math/aarch64/experimental/log10_2u.c"
+    "${AOR_SOURCE_DIR}/math/aarch64/experimental/log10_data.c"
+
+    "${AOR_SOURCE_DIR}/string/aarch64/memcpy-advsimd.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/memcmp.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/memchr-mte.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/strlen-mte.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/strcmp.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/strchr-mte.S"
+    "${AOR_SOURCE_DIR}/string/aarch64/strnlen.S"
+)
+
+add_library(aor ${SRCS})
+set_target_properties(aor PROPERTIES FOLDER "contrib/optimized-routines-cmake")
+
+# AOR asm entry points carry internal names (__memcpy_aarch64_simd, ...);
+# rename them to the public symbols during preprocessing of the .S files.
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/memcpy-advsimd.S"
+    PROPERTIES COMPILE_DEFINITIONS "__memcpy_aarch64_simd=memcpy;__memmove_aarch64_simd=memmove")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/memcmp.S"
+    PROPERTIES COMPILE_DEFINITIONS "__memcmp_aarch64=memcmp")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/memchr-mte.S"
+    PROPERTIES COMPILE_DEFINITIONS "__memchr_aarch64_mte=memchr")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/strlen-mte.S"
+    PROPERTIES COMPILE_DEFINITIONS "__strlen_aarch64_mte=strlen")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/strcmp.S"
+    PROPERTIES COMPILE_DEFINITIONS "__strcmp_aarch64=strcmp")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/strchr-mte.S"
+    PROPERTIES COMPILE_DEFINITIONS "__strchr_aarch64_mte=strchr")
+set_source_files_properties("${AOR_SOURCE_DIR}/string/aarch64/strnlen.S"
+    PROPERTIES COMPILE_DEFINITIONS "__strnlen_aarch64=strnlen")
+
+# The atan/atan2/log10 scalars live under math/aarch64/experimental and their
+# support structures in math_config.h are guarded by WANT_EXPERIMENTAL_MATH.
+target_compile_definitions(aor PRIVATE WANT_EXPERIMENTAL_MATH=1)
+
+# WANT_ROUNDING/WANT_ERRNO/HAVE_FAST_ROUND/HAVE_FAST_FMA take their correct
+# values from math_config.h defaults on aarch64. The test_sig.h/test_defs.h
+# includes in the sources are no-ops outside the AOR test harness but must
+# resolve, hence math/include.
+target_include_directories(aor PRIVATE
+    "${AOR_SOURCE_DIR}/math"
+    "${AOR_SOURCE_DIR}/math/include"
+    "${AOR_SOURCE_DIR}/string/aarch64"
+)
+
+target_compile_options(aor PRIVATE -ffreestanding -fno-builtin)