mirror of
https://github.com/QuantumNous/new-api.git
synced 2026-10-28 08:35:33 +08:00
fix(middleware): 过载哨兵改用容器口径并加入连续性与新鲜度判定 (#7317)
* fix(middleware): 过载哨兵改用容器口径并加入连续性与新鲜度判定 checkSystemPerformance 依赖 common.SystemStatus,而 updateSystemStatus 用 cpu.Percent(0,false) 与 mem.VirtualMemory() 采样,两者都是宿主机口径 (/proc/stat、/proc/meminfo),不具备 cgroup 感知。容器化部署下度量的是 宿主机而非本进程:容器配额 2 核时 /proc/stat 仍暴露宿主机全部核心,同节点 其它工作负载会把该值推高,导致正常 relay 流量被 503 system_cpu_overloaded 拒绝。同时该判定对单次采样即时生效,瞬时尖峰即可熔断。 本次调整: - 新增 common/container_usage.go:优先按容器 cgroup 配额计算自身 CPU 与内存 占用(v2 cpu.max/cpu.stat/memory.max/memory.current,兼容 v1),容器口径 不可用时回退宿主机口径; - 过载判定要求连续 3 次采样(约 15 秒)超阈值,任一采样回落即恢复; - 采样超过 30 秒未刷新时哨兵不再熔断,避免状态陈旧误伤流量; - 中间件改为消费采样结果中的过载标记,错误码与 503 语义不变。 验证:go build ./...、go test ./common/... ./middleware/... 通过;新增 容器口径与判定行为单测。 * fix(monitor): correct container CPU and memory overload detection --------- Co-authored-by: seefs001 <i@seefs.me>
This commit is contained in:
1 parent
d032204733
commit
09109ec315
4 files changed
+556
-45
No files matched your search
@@ -0,0 +1,174 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 容器自身资源采样。
|
||||
//
|
||||
// 历史实现直接读取宿主机口径(/proc/stat 与 /proc/meminfo),把同节点其它
|
||||
// 工作负载的占用也算成本进程过载,导致正常的 relay 流量被 503 熔断。这里
|
||||
// 改为优先读取当前容器的 cgroup 配额与用量,只有在拿不到 cgroup 数据时才由
|
||||
// 调用方回退到宿主机口径。
|
||||
|
||||
const (
|
||||
defaultCgroupRoot = "/sys/fs/cgroup"
|
||||
// cgroup v1 用接近 int64 上限的值表示"未限制"。
|
||||
unlimitedMemoryBytes = uint64(1) << 60
|
||||
)
|
||||
|
||||
// containerUsage 记录上一次采样点,用来计算区间内的 CPU 占用。
|
||||
type containerUsage struct {
|
||||
cgroupRoot string
|
||||
// schedulableCPUs 是本进程可调度的逻辑 CPU 数(Linux 上含 cpuset/affinity 限制)。
|
||||
schedulableCPUs float64
|
||||
cpuLastUsec uint64
|
||||
cpuLastSampled time.Time
|
||||
}
|
||||
|
||||
func newContainerUsage() *containerUsage {
|
||||
// runtime.NumCPU 在 Linux 上于进程启动时按 sched_getaffinity 计数,不受 GOMAXPROCS 影响。
|
||||
return &containerUsage{cgroupRoot: defaultCgroupRoot, schedulableCPUs: float64(runtime.NumCPU())}
|
||||
}
|
||||
|
||||
// cpuSaturation 返回相对容器可用 CPU 的使用率(百分比)。
|
||||
// 可用 CPU 取 cgroup 配额与可调度 CPU 的较小者:配额大于 cpuset 或节点核数时,
|
||||
// 按配额计算会让容器打满也无法触发保护。
|
||||
// ok=false 表示当前环境没有可用的 cgroup CPU 数据,调用方应回退到宿主机口径。
|
||||
func (u *containerUsage) cpuSaturation(now time.Time) (float64, bool) {
|
||||
limitCores, ok := u.readCgroupCPUQuotaCores()
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if u.schedulableCPUs > 0 {
|
||||
limitCores = min(limitCores, u.schedulableCPUs)
|
||||
}
|
||||
usageUsec, ok := u.readCgroupCPUUsageUsec()
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if u.cpuLastSampled.IsZero() || usageUsec < u.cpuLastUsec || !now.After(u.cpuLastSampled) {
|
||||
u.cpuLastUsec = usageUsec
|
||||
u.cpuLastSampled = now
|
||||
return 0, false
|
||||
}
|
||||
elapsed := now.Sub(u.cpuLastSampled)
|
||||
if elapsed < time.Second {
|
||||
return 0, false
|
||||
}
|
||||
deltaUsec := usageUsec - u.cpuLastUsec
|
||||
u.cpuLastUsec = usageUsec
|
||||
u.cpuLastSampled = now
|
||||
|
||||
usedCores := float64(deltaUsec) / 1e6 / elapsed.Seconds()
|
||||
return usedCores / limitCores * 100, true
|
||||
}
|
||||
|
||||
// memoryUsage 返回容器内存工作集相对上限的使用率(百分比)。
|
||||
// 工作集 = 用量 - 非活跃文件缓存,与 docker stats、kubelet 一致:用量包含可回收的
|
||||
// 页缓存,写日志或文件后会长期贴近上限,不代表内存压力。
|
||||
// ok=false 时调用方回退宿主机口径(同样不含缓存):内存未限制、文件不可读、
|
||||
// 或 memory.stat 缺少非活跃缓存字段。宁可回退旧口径,也不把含缓存的用量当作压力。
|
||||
func (u *containerUsage) memoryUsage() (float64, bool) {
|
||||
usagePath := filepath.Join(u.cgroupRoot, "memory.current")
|
||||
limitPath := filepath.Join(u.cgroupRoot, "memory.max")
|
||||
statPath := filepath.Join(u.cgroupRoot, "memory.stat")
|
||||
inactiveFileKey := "inactive_file"
|
||||
if _, err := os.Stat(usagePath); err != nil {
|
||||
v1Dir := filepath.Join(u.cgroupRoot, "memory")
|
||||
usagePath = filepath.Join(v1Dir, "memory.usage_in_bytes")
|
||||
limitPath = filepath.Join(v1Dir, "memory.limit_in_bytes")
|
||||
statPath = filepath.Join(v1Dir, "memory.stat")
|
||||
inactiveFileKey = "total_inactive_file"
|
||||
}
|
||||
|
||||
// cgroup v2 未限制时 memory.max 为 "max",解析失败即视为不可用。
|
||||
limit, ok := readCgroupUint(limitPath)
|
||||
if !ok || limit == 0 || limit >= unlimitedMemoryBytes {
|
||||
return 0, false
|
||||
}
|
||||
usage, ok := readCgroupUint(usagePath)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
inactiveFile, ok := readCgroupStatValue(statPath, inactiveFileKey)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
// 用量与 memory.stat 不是同一时刻的快照,缓存可能短暂大于用量,按 0 处理防止下溢。
|
||||
workingSet := usage - min(inactiveFile, usage)
|
||||
return float64(workingSet) / float64(limit) * 100, true
|
||||
}
|
||||
|
||||
// readCgroupCPUQuotaCores 返回 cgroup CPU 配额(核)。ok=false 表示未限制或不可读。
|
||||
func (u *containerUsage) readCgroupCPUQuotaCores() (float64, bool) {
|
||||
if raw, err := os.ReadFile(filepath.Join(u.cgroupRoot, "cpu.max")); err == nil {
|
||||
quotaText, periodText, found := strings.Cut(strings.TrimSpace(string(raw)), " ")
|
||||
if !found || quotaText == "max" {
|
||||
return 0, false
|
||||
}
|
||||
quota, err := strconv.ParseUint(quotaText, 10, 64)
|
||||
if err != nil || quota == 0 {
|
||||
return 0, false
|
||||
}
|
||||
period, err := strconv.ParseUint(strings.TrimSpace(periodText), 10, 64)
|
||||
if err != nil || period == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(quota) / float64(period), true
|
||||
}
|
||||
|
||||
// cgroup v1 未限制时 cfs_quota_us 为 -1,解析失败即视为不可用。
|
||||
v1Dir := filepath.Join(u.cgroupRoot, "cpu")
|
||||
quota, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_quota_us"))
|
||||
if !ok || quota == 0 {
|
||||
return 0, false
|
||||
}
|
||||
period, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_period_us"))
|
||||
if !ok || period == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(quota) / float64(period), true
|
||||
}
|
||||
|
||||
func (u *containerUsage) readCgroupCPUUsageUsec() (uint64, bool) {
|
||||
statPath := filepath.Join(u.cgroupRoot, "cpu.stat")
|
||||
if _, err := os.Stat(statPath); err == nil {
|
||||
return readCgroupStatValue(statPath, "usage_usec")
|
||||
}
|
||||
// cgroup v1 的 cpuacct.usage 单位是纳秒。
|
||||
nanos, ok := readCgroupUint(filepath.Join(u.cgroupRoot, "cpuacct", "cpuacct.usage"))
|
||||
return nanos / 1000, ok
|
||||
}
|
||||
|
||||
// readCgroupUint 读取只含一个无符号整数的 cgroup 文件。
|
||||
func readCgroupUint(path string) (uint64, bool) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64)
|
||||
return value, err == nil
|
||||
}
|
||||
|
||||
// readCgroupStatValue 从 "key value" 逐行格式的 cgroup 统计文件(cpu.stat、memory.stat)中读取一项。
|
||||
func readCgroupStatValue(path string, key string) (uint64, bool) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for line := range strings.SplitSeq(string(raw), "\n") {
|
||||
name, valueText, found := strings.Cut(line, " ")
|
||||
if !found || name != key {
|
||||
continue
|
||||
}
|
||||
value, err := strconv.ParseUint(strings.TrimSpace(valueText), 10, 64)
|
||||
return value, err == nil
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
+96
-17
@@ -1,6 +1,7 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -25,10 +26,36 @@ type SystemStatus struct {
|
||||
CPUUsage float64
|
||||
MemoryUsage float64
|
||||
DiskUsage float64
|
||||
SampledAt time.Time
|
||||
// 过载判定带连续性要求:单一采样尖峰不再直接熔断,需连续多次超阈值。
|
||||
CPUOverloaded bool
|
||||
MemoryOverloaded bool
|
||||
DiskOverloaded bool
|
||||
}
|
||||
|
||||
// SystemOverload 是哨兵据以拒绝请求的过载项,Error 文本带当前值与生效阈值。
|
||||
type SystemOverload struct {
|
||||
Code string
|
||||
Resource string
|
||||
Usage float64
|
||||
Threshold int
|
||||
}
|
||||
|
||||
func (o *SystemOverload) Error() string {
|
||||
return fmt.Sprintf("system %s overloaded (current: %.1f%%, threshold: %d%%)", o.Resource, o.Usage, o.Threshold)
|
||||
}
|
||||
|
||||
var latestSystemStatus atomic.Value
|
||||
|
||||
const (
|
||||
systemMonitorInterval = 5 * time.Second
|
||||
// 连续 3 次采样(约 15 秒)仍高于阈值才判定过载,
|
||||
// 任一采样回落到阈值以下即恢复,避免瞬时尖峰打满重试。
|
||||
overloadConsecutiveSamples = 3
|
||||
// 采样超过该时长未刷新(监控未运行或被阻塞)时,哨兵不据此熔断。
|
||||
systemStatusMaxAge = 30 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
latestSystemStatus.Store(SystemStatus{})
|
||||
}
|
||||
@@ -36,46 +63,98 @@ func init() {
|
||||
// StartSystemMonitor 启动系统监控
|
||||
func StartSystemMonitor() {
|
||||
go func() {
|
||||
usage := newContainerUsage()
|
||||
var detector overloadDetector
|
||||
for {
|
||||
config := GetPerformanceMonitorConfig()
|
||||
if !config.Enabled {
|
||||
time.Sleep(30 * time.Second)
|
||||
detector = overloadDetector{}
|
||||
latestSystemStatus.Store(SystemStatus{SampledAt: time.Now()})
|
||||
time.Sleep(systemMonitorInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
updateSystemStatus()
|
||||
time.Sleep(5 * time.Second)
|
||||
status := collectSystemStatus(usage)
|
||||
detector.mark(&status, config)
|
||||
latestSystemStatus.Store(status)
|
||||
|
||||
time.Sleep(systemMonitorInterval)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func updateSystemStatus() {
|
||||
var status SystemStatus
|
||||
// overloadDetector 为每项资源累计连续超阈值的采样次数:连续 overloadConsecutiveSamples 次
|
||||
// 才标记过载,任一次回落立即清零并重新累计。
|
||||
type overloadDetector struct {
|
||||
cpuSamples int
|
||||
memorySamples int
|
||||
diskSamples int
|
||||
}
|
||||
|
||||
// CPU
|
||||
// 注意:cpu.Percent(0, false) 返回自上次调用以来的 CPU 使用率
|
||||
// 如果是第一次调用,可能会返回错误或不准确的值,但在循环中会逐渐正常
|
||||
percents, err := cpu.Percent(0, false)
|
||||
if err == nil && len(percents) > 0 {
|
||||
func (d *overloadDetector) mark(status *SystemStatus, config PerformanceMonitorConfig) {
|
||||
status.CPUOverloaded = countOverloadSample(&d.cpuSamples, exceedsThreshold(status.CPUUsage, config.CPUThreshold))
|
||||
status.MemoryOverloaded = countOverloadSample(&d.memorySamples, exceedsThreshold(status.MemoryUsage, config.MemoryThreshold))
|
||||
status.DiskOverloaded = countOverloadSample(&d.diskSamples, exceedsThreshold(status.DiskUsage, config.DiskThreshold))
|
||||
}
|
||||
|
||||
func countOverloadSample(samples *int, over bool) bool {
|
||||
if !over {
|
||||
*samples = 0
|
||||
return false
|
||||
}
|
||||
*samples = min(*samples+1, overloadConsecutiveSamples)
|
||||
return *samples == overloadConsecutiveSamples
|
||||
}
|
||||
|
||||
// exceedsThreshold 判断采样值是否超过阈值;阈值 <= 0 表示不检查该项。
|
||||
func exceedsThreshold(usage float64, threshold int) bool {
|
||||
return threshold > 0 && int(usage) > threshold
|
||||
}
|
||||
|
||||
func collectSystemStatus(usage *containerUsage) SystemStatus {
|
||||
now := time.Now()
|
||||
status := SystemStatus{SampledAt: now}
|
||||
|
||||
// CPU / 内存优先按容器 cgroup 配额计算;容器口径不可用时回退宿主机口径,
|
||||
// 保持对未限制资源或被挂载为宿主 cgroup 的环境的兼容。
|
||||
if saturation, ok := usage.cpuSaturation(now); ok {
|
||||
status.CPUUsage = saturation
|
||||
} else if percents, err := cpu.Percent(0, false); err == nil && len(percents) > 0 {
|
||||
status.CPUUsage = percents[0]
|
||||
}
|
||||
|
||||
// Memory
|
||||
memInfo, err := mem.VirtualMemory()
|
||||
if err == nil {
|
||||
if containerMemory, ok := usage.memoryUsage(); ok {
|
||||
status.MemoryUsage = containerMemory
|
||||
} else if memInfo, err := mem.VirtualMemory(); err == nil {
|
||||
status.MemoryUsage = memInfo.UsedPercent
|
||||
}
|
||||
|
||||
// Disk
|
||||
diskInfo := GetDiskSpaceInfo()
|
||||
if diskInfo.Total > 0 {
|
||||
status.DiskUsage = diskInfo.UsedPercent
|
||||
}
|
||||
|
||||
latestSystemStatus.Store(status)
|
||||
return status
|
||||
}
|
||||
|
||||
// GetSystemStatus 获取当前系统状态
|
||||
func GetSystemStatus() SystemStatus {
|
||||
return latestSystemStatus.Load().(SystemStatus)
|
||||
}
|
||||
|
||||
// CheckSystemOverload 返回哨兵应据以熔断的过载项。监控关闭、采样缺失或已超过
|
||||
// systemStatusMaxAge 时返回 nil。过载标记来自连续采样,阈值按当前配置复核,
|
||||
// 运行时关闭或调高阈值后立即放行,不必等待下一次采样。
|
||||
func CheckSystemOverload(status SystemStatus, config PerformanceMonitorConfig, now time.Time) *SystemOverload {
|
||||
if !config.Enabled || status.SampledAt.IsZero() || now.Sub(status.SampledAt) > systemStatusMaxAge {
|
||||
return nil
|
||||
}
|
||||
if status.CPUOverloaded && exceedsThreshold(status.CPUUsage, config.CPUThreshold) {
|
||||
return &SystemOverload{Code: "system_cpu_overloaded", Resource: "cpu", Usage: status.CPUUsage, Threshold: config.CPUThreshold}
|
||||
}
|
||||
if status.MemoryOverloaded && exceedsThreshold(status.MemoryUsage, config.MemoryThreshold) {
|
||||
return &SystemOverload{Code: "system_memory_overloaded", Resource: "memory", Usage: status.MemoryUsage, Threshold: config.MemoryThreshold}
|
||||
}
|
||||
if status.DiskOverloaded && exceedsThreshold(status.DiskUsage, config.DiskThreshold) {
|
||||
return &SystemOverload{Code: "system_disk_overloaded", Resource: "disk", Usage: status.DiskUsage, Threshold: config.DiskThreshold}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newTestContainerUsage 在临时目录中构造 cgroup 文件,避免依赖运行环境的真实 cgroup。
|
||||
func newTestContainerUsage(t *testing.T, schedulableCPUs float64, files map[string]string) *containerUsage {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, content := range files {
|
||||
writeTestCgroupFile(t, filepath.Join(root, name), content)
|
||||
}
|
||||
return &containerUsage{cgroupRoot: root, schedulableCPUs: schedulableCPUs}
|
||||
}
|
||||
|
||||
func writeTestCgroupFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
}
|
||||
|
||||
func TestContainerCPUSaturation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
schedulableCPUs float64
|
||||
files map[string]string
|
||||
usageFile string
|
||||
usageAfter string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "cgroup v2 quota",
|
||||
schedulableCPUs: 4,
|
||||
files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\nuser_usec 900000\n"},
|
||||
usageFile: "cpu.stat",
|
||||
usageAfter: "usage_usec 6000000\n", // 5 秒用 1 核,配额 2 核
|
||||
want: 50,
|
||||
},
|
||||
{
|
||||
name: "fractional quota",
|
||||
schedulableCPUs: 4,
|
||||
files: map[string]string{"cpu.max": "50000 100000\n", "cpu.stat": "usage_usec 1000000\n"},
|
||||
usageFile: "cpu.stat",
|
||||
usageAfter: "usage_usec 3500000\n", // 5 秒用 0.5 核,配额 0.5 核
|
||||
want: 100,
|
||||
},
|
||||
{
|
||||
name: "quota above schedulable CPUs",
|
||||
schedulableCPUs: 1,
|
||||
files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\n"},
|
||||
usageFile: "cpu.stat",
|
||||
usageAfter: "usage_usec 6000000\n", // cpuset 只有 1 核,已打满
|
||||
want: 100,
|
||||
},
|
||||
{
|
||||
name: "cgroup v1 quota",
|
||||
schedulableCPUs: 4,
|
||||
files: map[string]string{
|
||||
"cpu/cpu.cfs_quota_us": "200000\n",
|
||||
"cpu/cpu.cfs_period_us": "100000\n",
|
||||
"cpuacct/cpuacct.usage": "1000000000\n",
|
||||
},
|
||||
usageFile: "cpuacct/cpuacct.usage",
|
||||
usageAfter: "11000000000\n", // 纳秒:5 秒用 2 核
|
||||
want: 100,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
usage := newTestContainerUsage(t, tt.schedulableCPUs, tt.files)
|
||||
base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
|
||||
_, ok := usage.cpuSaturation(base)
|
||||
assert.False(t, ok, "首次采样只建立基线")
|
||||
|
||||
writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, tt.usageFile), tt.usageAfter)
|
||||
got, ok := usage.cpuSaturation(base.Add(5 * time.Second))
|
||||
require.True(t, ok)
|
||||
assert.InDelta(t, tt.want, got, 0.01)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerCPUSaturationFallsBackWithoutQuota(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
}{
|
||||
{name: "cgroup v2 unlimited", files: map[string]string{"cpu.max": "max 100000\n", "cpu.stat": "usage_usec 1000000\n"}},
|
||||
{name: "cgroup v1 unlimited", files: map[string]string{"cpu/cpu.cfs_quota_us": "-1\n", "cpu/cpu.cfs_period_us": "100000\n", "cpuacct/cpuacct.usage": "1000\n"}},
|
||||
{name: "no cgroup files"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
usage := newTestContainerUsage(t, 4, tt.files)
|
||||
base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
|
||||
usage.cpuSaturation(base)
|
||||
_, ok := usage.cpuSaturation(base.Add(5 * time.Second))
|
||||
assert.False(t, ok, "没有配额时交由调用方回退宿主机口径")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerMemoryUsageUsesWorkingSet(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "cgroup v2 excludes inactive file cache",
|
||||
files: map[string]string{
|
||||
"memory.current": "266338304\n",
|
||||
"memory.max": "268435456\n",
|
||||
"memory.stat": "anon 4435968\nfile 253362176\ninactive_file 251662336\n",
|
||||
},
|
||||
want: 5.4672,
|
||||
},
|
||||
{
|
||||
name: "cgroup v2 high working set keeps protection",
|
||||
files: map[string]string{
|
||||
"memory.current": "255852544\n",
|
||||
"memory.max": "268435456\n",
|
||||
"memory.stat": "anon 254803968\ninactive_file 1048576\n",
|
||||
},
|
||||
want: 94.9219,
|
||||
},
|
||||
{
|
||||
name: "cgroup v1 uses hierarchical inactive file cache",
|
||||
files: map[string]string{
|
||||
"memory/memory.usage_in_bytes": "805306368\n",
|
||||
"memory/memory.limit_in_bytes": "1610612736\n",
|
||||
"memory/memory.stat": "inactive_file 999\ntotal_inactive_file 268435456\n",
|
||||
},
|
||||
want: 33.3333,
|
||||
},
|
||||
{
|
||||
name: "cache larger than usage does not underflow",
|
||||
files: map[string]string{
|
||||
"memory.current": "1000\n",
|
||||
"memory.max": "268435456\n",
|
||||
"memory.stat": "inactive_file 2000\n",
|
||||
},
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := newTestContainerUsage(t, 4, tt.files).memoryUsage()
|
||||
require.True(t, ok)
|
||||
assert.InDelta(t, tt.want, got, 0.001)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerMemoryUsageFallsBackWhenUnavailable(t *testing.T) {
|
||||
limited := map[string]string{"memory.current": "266338304\n", "memory.max": "268435456\n"}
|
||||
tests := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
stat string
|
||||
}{
|
||||
{name: "cgroup v2 unlimited", files: map[string]string{"memory.current": "1000\n", "memory.max": "max\n"}, stat: "inactive_file 0\n"},
|
||||
{name: "cgroup v1 unlimited", files: map[string]string{
|
||||
"memory/memory.usage_in_bytes": "1000\n",
|
||||
"memory/memory.limit_in_bytes": "9223372036854771712\n",
|
||||
"memory/memory.stat": "total_inactive_file 0\n",
|
||||
}},
|
||||
{name: "memory.stat missing", files: limited},
|
||||
{name: "inactive file field missing", files: limited, stat: "anon 4435968\n"},
|
||||
{name: "inactive file field corrupt", files: limited, stat: "inactive_file abc\n"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
usage := newTestContainerUsage(t, 4, tt.files)
|
||||
if tt.stat != "" {
|
||||
writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, "memory.stat"), tt.stat)
|
||||
}
|
||||
_, ok := usage.memoryUsage()
|
||||
assert.False(t, ok, "拿不到可信工作集时交由调用方回退宿主机口径")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemOverloadRequiresConsecutiveSamples(t *testing.T) {
|
||||
config := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95}
|
||||
now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
|
||||
cpuSamples := []struct {
|
||||
usage float64
|
||||
wantOverload bool
|
||||
}{
|
||||
{usage: 95, wantOverload: false},
|
||||
{usage: 95, wantOverload: false},
|
||||
{usage: 95, wantOverload: true},
|
||||
{usage: 95, wantOverload: true},
|
||||
{usage: 50, wantOverload: false}, // 回落即恢复
|
||||
{usage: 95, wantOverload: false}, // 重新累计
|
||||
{usage: 95, wantOverload: false},
|
||||
{usage: 95, wantOverload: true},
|
||||
}
|
||||
|
||||
var detector overloadDetector
|
||||
for i, sample := range cpuSamples {
|
||||
status := SystemStatus{CPUUsage: sample.usage, SampledAt: now}
|
||||
detector.mark(&status, config)
|
||||
overload := CheckSystemOverload(status, config, now)
|
||||
if !sample.wantOverload {
|
||||
assert.Nil(t, overload, "sample %d", i+1)
|
||||
continue
|
||||
}
|
||||
require.NotNil(t, overload, "sample %d", i+1)
|
||||
assert.Equal(t, "system_cpu_overloaded", overload.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSystemOverload(t *testing.T) {
|
||||
now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
|
||||
enabled := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95}
|
||||
cpuOverloaded := SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now}
|
||||
tests := []struct {
|
||||
name string
|
||||
status SystemStatus
|
||||
config PerformanceMonitorConfig
|
||||
wantCode string
|
||||
wantMessage string
|
||||
}{
|
||||
{
|
||||
name: "fresh cpu overload",
|
||||
status: cpuOverloaded,
|
||||
config: enabled,
|
||||
wantCode: "system_cpu_overloaded",
|
||||
wantMessage: "system cpu overloaded (current: 99.5%, threshold: 90%)",
|
||||
},
|
||||
{
|
||||
name: "fresh memory overload",
|
||||
status: SystemStatus{MemoryUsage: 96.2, MemoryOverloaded: true, SampledAt: now},
|
||||
config: enabled,
|
||||
wantCode: "system_memory_overloaded",
|
||||
wantMessage: "system memory overloaded (current: 96.2%, threshold: 90%)",
|
||||
},
|
||||
{
|
||||
name: "fresh disk overload",
|
||||
status: SystemStatus{DiskUsage: 97, DiskOverloaded: true, SampledAt: now},
|
||||
config: enabled,
|
||||
wantCode: "system_disk_overloaded",
|
||||
wantMessage: "system disk overloaded (current: 97.0%, threshold: 95%)",
|
||||
},
|
||||
{
|
||||
name: "lowered threshold reports the active threshold",
|
||||
status: cpuOverloaded,
|
||||
config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 80},
|
||||
wantCode: "system_cpu_overloaded",
|
||||
wantMessage: "system cpu overloaded (current: 99.5%, threshold: 80%)",
|
||||
},
|
||||
{name: "monitor disabled", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: false, CPUThreshold: 90}},
|
||||
{name: "cpu threshold disabled at runtime", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 0}},
|
||||
{name: "cpu threshold raised above usage", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 100}},
|
||||
{name: "stale sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now.Add(-31 * time.Second)}, config: enabled},
|
||||
{name: "missing sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true}, config: enabled},
|
||||
{name: "usage above threshold without consecutive samples", status: SystemStatus{CPUUsage: 99.5, SampledAt: now}, config: enabled},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
overload := CheckSystemOverload(tt.status, tt.config, now)
|
||||
if tt.wantCode == "" {
|
||||
assert.Nil(t, overload)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, overload)
|
||||
assert.Equal(t, tt.wantCode, overload.Code)
|
||||
assert.Equal(t, tt.wantMessage, overload.Error())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/types"
|
||||
@@ -39,33 +39,11 @@ func SystemPerformanceCheck() gin.HandlerFunc {
|
||||
|
||||
// checkSystemPerformance 检查系统性能是否超过阈值
|
||||
func checkSystemPerformance() *types.NewAPIError {
|
||||
config := common.GetPerformanceMonitorConfig()
|
||||
if !config.Enabled {
|
||||
// 过载标记在采样侧完成:CPU / 内存按容器 cgroup 配额计算,并要求连续多次
|
||||
// 超阈值;判定时再按当前阈值复核,并忽略过期采样。
|
||||
overload := common.CheckSystemOverload(common.GetSystemStatus(), common.GetPerformanceMonitorConfig(), time.Now())
|
||||
if overload == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
status := common.GetSystemStatus()
|
||||
|
||||
// 检查 CPU
|
||||
if config.CPUThreshold > 0 && int(status.CPUUsage) > config.CPUThreshold {
|
||||
return types.NewErrorWithStatusCode(
|
||||
fmt.Errorf("system cpu overloaded (current: %.1f%%, threshold: %d%%)", status.CPUUsage, config.CPUThreshold),
|
||||
"system_cpu_overloaded", http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
// 检查内存
|
||||
if config.MemoryThreshold > 0 && int(status.MemoryUsage) > config.MemoryThreshold {
|
||||
return types.NewErrorWithStatusCode(
|
||||
fmt.Errorf("system memory overloaded (current: %.1f%%, threshold: %d%%)", status.MemoryUsage, config.MemoryThreshold),
|
||||
"system_memory_overloaded", http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
// 检查磁盘
|
||||
if config.DiskThreshold > 0 && int(status.DiskUsage) > config.DiskThreshold {
|
||||
return types.NewErrorWithStatusCode(
|
||||
fmt.Errorf("system disk overloaded (current: %.1f%%, threshold: %d%%)", status.DiskUsage, config.DiskThreshold),
|
||||
"system_disk_overloaded", http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
return nil
|
||||
return types.NewErrorWithStatusCode(overload, types.ErrorCode(overload.Code), http.StatusServiceUnavailable)
|
||||
}
|
||||
Reference in new issue
Block a user