mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-10-26 10:57:38 +08:00
Zhan Xusheng
dcd65074e2
The vDSO basetimes for CLOCK_MONOTONIC and CLOCK_BOOTTIME are stored in the scaled nanoseconds of tkr_mono, so normalising them requires a division by NSEC_PER_SEC << shift. That divisor does not fit the u32 parameter of __iter_div_u64_rem(), so update_vdso_time_data() open-codes the same iterative division twice. Repeated subtraction is the appropriate form at these two sites because the quotient never exceeds one. accumulate_nsecs_to_secs() keeps xtime_nsec below one scaled second, and the offset added to it is a normalised timespec64 fraction, so the dividend stays below twice the divisor. That bound holds on every architecture, which matters more here than the cost of a division on any particular one. Add __iter_div64_u64_rem(), the u64-divisor counterpart of __iter_div_u64_rem(), and use it at both sites. Return the quotient as a u32 for consistency with the u32-divisor variant; the bound above leaves no use for a wider type. Store the remainder directly into the basetime, as the coarse clocks already do. The CLOCK_BOOTTIME copy of the CLOCK_MONOTONIC values then takes both fields from the same place. Folding the two open-coded loops into one inlined helper removes 16 bytes of vsyscall.o text on x86-64 with gcc 13. No functional change. Suggested-by: David Laight <david.laight.linux@gmail.com> Signed-off-by: Zhan Xusheng <zhanxusheng@xiaomi.com> Signed-off-by: Thomas Gleixner <tglx@kernel.org> Reviewed-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de> Link: https://patch.msgid.link/20261002153104.1347978-3-zhanxusheng@xiaomi.com
86 lines
1.6 KiB
C
86 lines
1.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __VDSO_MATH64_H
|
|
#define __VDSO_MATH64_H
|
|
|
|
#include <linux/compiler.h>
|
|
|
|
static __always_inline u32 __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
|
|
{
|
|
u32 ret = 0;
|
|
|
|
while (dividend >= divisor) {
|
|
/*
|
|
* Prevent the compiler from optimising this loop into a
|
|
* modulo operation.
|
|
*/
|
|
OPTIMIZER_HIDE_VAR(dividend);
|
|
|
|
dividend -= divisor;
|
|
ret++;
|
|
}
|
|
|
|
*remainder = dividend;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static __always_inline u32 __iter_div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
|
|
{
|
|
u32 ret = 0;
|
|
|
|
while (dividend >= divisor) {
|
|
/*
|
|
* Prevent the compiler from optimising this loop into a
|
|
* modulo operation.
|
|
*/
|
|
OPTIMIZER_HIDE_VAR(dividend);
|
|
|
|
dividend -= divisor;
|
|
ret++;
|
|
}
|
|
|
|
*remainder = dividend;
|
|
|
|
return ret;
|
|
}
|
|
|
|
#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
|
|
|
|
#ifndef mul_u64_u32_add_u64_shr
|
|
static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
|
|
{
|
|
return (u64)((((unsigned __int128)a * mul) + b) >> shift);
|
|
}
|
|
#endif /* mul_u64_u32_add_u64_shr */
|
|
|
|
#else
|
|
|
|
#ifndef mul_u64_u32_add_u64_shr
|
|
#ifndef mul_u32_u32
|
|
static inline u64 mul_u32_u32(u32 a, u32 b)
|
|
{
|
|
return (u64)a * b;
|
|
}
|
|
#define mul_u32_u32 mul_u32_u32
|
|
#endif
|
|
static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
|
|
{
|
|
u32 ah = a >> 32, al = a;
|
|
bool ovf;
|
|
u64 ret;
|
|
|
|
ovf = __builtin_add_overflow(mul_u32_u32(al, mul), b, &ret);
|
|
ret >>= shift;
|
|
if (ovf && shift)
|
|
ret += 1ULL << (64 - shift);
|
|
if (ah)
|
|
ret += mul_u32_u32(ah, mul) << (32 - shift);
|
|
|
|
return ret;
|
|
}
|
|
#endif /* mul_u64_u32_add_u64_shr */
|
|
|
|
#endif
|
|
|
|
#endif /* __VDSO_MATH64_H */
|