diff --git a/common/container_usage.go b/common/container_usage.go new file mode 100644 index 0000000000..0b43d8bd65 --- /dev/null +++ b/common/container_usage.go @@ -0,0 +1,174 @@ +package common + +import ( + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +// 容器自身资源采样。 +// +// 历史实现直接读取宿主机口径（/proc/stat 与 /proc/meminfo），把同节点其它 +// 工作负载的占用也算成本进程过载，导致正常的 relay 流量被 503 熔断。这里 +// 改为优先读取当前容器的 cgroup 配额与用量，只有在拿不到 cgroup 数据时才由 +// 调用方回退到宿主机口径。 + +const ( + defaultCgroupRoot = "/sys/fs/cgroup" + // cgroup v1 用接近 int64 上限的值表示"未限制"。 + unlimitedMemoryBytes = uint64(1) << 60 +) + +// containerUsage 记录上一次采样点，用来计算区间内的 CPU 占用。 +type containerUsage struct { + cgroupRoot string + // schedulableCPUs 是本进程可调度的逻辑 CPU 数（Linux 上含 cpuset/affinity 限制）。 + schedulableCPUs float64 + cpuLastUsec uint64 + cpuLastSampled time.Time +} + +func newContainerUsage() *containerUsage { + // runtime.NumCPU 在 Linux 上于进程启动时按 sched_getaffinity 计数，不受 GOMAXPROCS 影响。 + return &containerUsage{cgroupRoot: defaultCgroupRoot, schedulableCPUs: float64(runtime.NumCPU())} +} + +// cpuSaturation 返回相对容器可用 CPU 的使用率（百分比）。 +// 可用 CPU 取 cgroup 配额与可调度 CPU 的较小者：配额大于 cpuset 或节点核数时， +// 按配额计算会让容器打满也无法触发保护。 +// ok=false 表示当前环境没有可用的 cgroup CPU 数据，调用方应回退到宿主机口径。 +func (u *containerUsage) cpuSaturation(now time.Time) (float64, bool) { + limitCores, ok := u.readCgroupCPUQuotaCores() + if !ok { + return 0, false + } + if u.schedulableCPUs > 0 { + limitCores = min(limitCores, u.schedulableCPUs) + } + usageUsec, ok := u.readCgroupCPUUsageUsec() + if !ok { + return 0, false + } + if u.cpuLastSampled.IsZero() || usageUsec < u.cpuLastUsec || !now.After(u.cpuLastSampled) { + u.cpuLastUsec = usageUsec + u.cpuLastSampled = now + return 0, false + } + elapsed := now.Sub(u.cpuLastSampled) + if elapsed < time.Second { + return 0, false + } + deltaUsec := usageUsec - u.cpuLastUsec + u.cpuLastUsec = usageUsec + u.cpuLastSampled = now + + usedCores := float64(deltaUsec) / 1e6 / elapsed.Seconds() + return usedCores / limitCores * 100, true +} + +// memoryUsage 返回容器内存工作集相对上限的使用率（百分比）。 +// 工作集 = 用量 - 非活跃文件缓存，与 docker stats、kubelet 一致：用量包含可回收的 +// 页缓存，写日志或文件后会长期贴近上限，不代表内存压力。 +// ok=false 时调用方回退宿主机口径（同样不含缓存）：内存未限制、文件不可读、 +// 或 memory.stat 缺少非活跃缓存字段。宁可回退旧口径，也不把含缓存的用量当作压力。 +func (u *containerUsage) memoryUsage() (float64, bool) { + usagePath := filepath.Join(u.cgroupRoot, "memory.current") + limitPath := filepath.Join(u.cgroupRoot, "memory.max") + statPath := filepath.Join(u.cgroupRoot, "memory.stat") + inactiveFileKey := "inactive_file" + if _, err := os.Stat(usagePath); err != nil { + v1Dir := filepath.Join(u.cgroupRoot, "memory") + usagePath = filepath.Join(v1Dir, "memory.usage_in_bytes") + limitPath = filepath.Join(v1Dir, "memory.limit_in_bytes") + statPath = filepath.Join(v1Dir, "memory.stat") + inactiveFileKey = "total_inactive_file" + } + + // cgroup v2 未限制时 memory.max 为 "max"，解析失败即视为不可用。 + limit, ok := readCgroupUint(limitPath) + if !ok || limit == 0 || limit >= unlimitedMemoryBytes { + return 0, false + } + usage, ok := readCgroupUint(usagePath) + if !ok { + return 0, false + } + inactiveFile, ok := readCgroupStatValue(statPath, inactiveFileKey) + if !ok { + return 0, false + } + // 用量与 memory.stat 不是同一时刻的快照，缓存可能短暂大于用量，按 0 处理防止下溢。 + workingSet := usage - min(inactiveFile, usage) + return float64(workingSet) / float64(limit) * 100, true +} + +// readCgroupCPUQuotaCores 返回 cgroup CPU 配额（核）。ok=false 表示未限制或不可读。 +func (u *containerUsage) readCgroupCPUQuotaCores() (float64, bool) { + if raw, err := os.ReadFile(filepath.Join(u.cgroupRoot, "cpu.max")); err == nil { + quotaText, periodText, found := strings.Cut(strings.TrimSpace(string(raw)), " ") + if !found || quotaText == "max" { + return 0, false + } + quota, err := strconv.ParseUint(quotaText, 10, 64) + if err != nil || quota == 0 { + return 0, false + } + period, err := strconv.ParseUint(strings.TrimSpace(periodText), 10, 64) + if err != nil || period == 0 { + return 0, false + } + return float64(quota) / float64(period), true + } + + // cgroup v1 未限制时 cfs_quota_us 为 -1，解析失败即视为不可用。 + v1Dir := filepath.Join(u.cgroupRoot, "cpu") + quota, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_quota_us")) + if !ok || quota == 0 { + return 0, false + } + period, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_period_us")) + if !ok || period == 0 { + return 0, false + } + return float64(quota) / float64(period), true +} + +func (u *containerUsage) readCgroupCPUUsageUsec() (uint64, bool) { + statPath := filepath.Join(u.cgroupRoot, "cpu.stat") + if _, err := os.Stat(statPath); err == nil { + return readCgroupStatValue(statPath, "usage_usec") + } + // cgroup v1 的 cpuacct.usage 单位是纳秒。 + nanos, ok := readCgroupUint(filepath.Join(u.cgroupRoot, "cpuacct", "cpuacct.usage")) + return nanos / 1000, ok +} + +// readCgroupUint 读取只含一个无符号整数的 cgroup 文件。 +func readCgroupUint(path string) (uint64, bool) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, false + } + value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64) + return value, err == nil +} + +// readCgroupStatValue 从 "key value" 逐行格式的 cgroup 统计文件（cpu.stat、memory.stat）中读取一项。 +func readCgroupStatValue(path string, key string) (uint64, bool) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, false + } + for line := range strings.SplitSeq(string(raw), "\n") { + name, valueText, found := strings.Cut(line, " ") + if !found || name != key { + continue + } + value, err := strconv.ParseUint(strings.TrimSpace(valueText), 10, 64) + return value, err == nil + } + return 0, false +} diff --git a/common/system_monitor.go b/common/system_monitor.go index 26710faede..a998562e62 100644 --- a/common/system_monitor.go +++ b/common/system_monitor.go @@ -1,6 +1,7 @@ package common import ( + "fmt" "sync/atomic" "time" @@ -25,10 +26,36 @@ type SystemStatus struct { CPUUsage float64 MemoryUsage float64 DiskUsage float64 + SampledAt time.Time + // 过载判定带连续性要求：单一采样尖峰不再直接熔断，需连续多次超阈值。 + CPUOverloaded bool + MemoryOverloaded bool + DiskOverloaded bool +} + +// SystemOverload 是哨兵据以拒绝请求的过载项，Error 文本带当前值与生效阈值。 +type SystemOverload struct { + Code string + Resource string + Usage float64 + Threshold int +} + +func (o *SystemOverload) Error() string { + return fmt.Sprintf("system %s overloaded (current: %.1f%%, threshold: %d%%)", o.Resource, o.Usage, o.Threshold) } var latestSystemStatus atomic.Value +const ( + systemMonitorInterval = 5 * time.Second + // 连续 3 次采样（约 15 秒）仍高于阈值才判定过载， + // 任一采样回落到阈值以下即恢复，避免瞬时尖峰打满重试。 + overloadConsecutiveSamples = 3 + // 采样超过该时长未刷新（监控未运行或被阻塞）时，哨兵不据此熔断。 + systemStatusMaxAge = 30 * time.Second +) + func init() { latestSystemStatus.Store(SystemStatus{}) } @@ -36,46 +63,98 @@ func init() { // StartSystemMonitor 启动系统监控 func StartSystemMonitor() { go func() { + usage := newContainerUsage() + var detector overloadDetector for { config := GetPerformanceMonitorConfig() if !config.Enabled { - time.Sleep(30 * time.Second) + detector = overloadDetector{} + latestSystemStatus.Store(SystemStatus{SampledAt: time.Now()}) + time.Sleep(systemMonitorInterval) continue } - updateSystemStatus() - time.Sleep(5 * time.Second) + status := collectSystemStatus(usage) + detector.mark(&status, config) + latestSystemStatus.Store(status) + + time.Sleep(systemMonitorInterval) } }() } -func updateSystemStatus() { - var status SystemStatus +// overloadDetector 为每项资源累计连续超阈值的采样次数：连续 overloadConsecutiveSamples 次 +// 才标记过载，任一次回落立即清零并重新累计。 +type overloadDetector struct { + cpuSamples int + memorySamples int + diskSamples int +} - // CPU - // 注意：cpu.Percent(0, false) 返回自上次调用以来的 CPU 使用率 - // 如果是第一次调用，可能会返回错误或不准确的值，但在循环中会逐渐正常 - percents, err := cpu.Percent(0, false) - if err == nil && len(percents) > 0 { +func (d *overloadDetector) mark(status *SystemStatus, config PerformanceMonitorConfig) { + status.CPUOverloaded = countOverloadSample(&d.cpuSamples, exceedsThreshold(status.CPUUsage, config.CPUThreshold)) + status.MemoryOverloaded = countOverloadSample(&d.memorySamples, exceedsThreshold(status.MemoryUsage, config.MemoryThreshold)) + status.DiskOverloaded = countOverloadSample(&d.diskSamples, exceedsThreshold(status.DiskUsage, config.DiskThreshold)) +} + +func countOverloadSample(samples *int, over bool) bool { + if !over { + *samples = 0 + return false + } + *samples = min(*samples+1, overloadConsecutiveSamples) + return *samples == overloadConsecutiveSamples +} + +// exceedsThreshold 判断采样值是否超过阈值；阈值 <= 0 表示不检查该项。 +func exceedsThreshold(usage float64, threshold int) bool { + return threshold > 0 && int(usage) > threshold +} + +func collectSystemStatus(usage *containerUsage) SystemStatus { + now := time.Now() + status := SystemStatus{SampledAt: now} + + // CPU / 内存优先按容器 cgroup 配额计算；容器口径不可用时回退宿主机口径， + // 保持对未限制资源或被挂载为宿主 cgroup 的环境的兼容。 + if saturation, ok := usage.cpuSaturation(now); ok { + status.CPUUsage = saturation + } else if percents, err := cpu.Percent(0, false); err == nil && len(percents) > 0 { status.CPUUsage = percents[0] } - - // Memory - memInfo, err := mem.VirtualMemory() - if err == nil { + if containerMemory, ok := usage.memoryUsage(); ok { + status.MemoryUsage = containerMemory + } else if memInfo, err := mem.VirtualMemory(); err == nil { status.MemoryUsage = memInfo.UsedPercent } - // Disk diskInfo := GetDiskSpaceInfo() if diskInfo.Total > 0 { status.DiskUsage = diskInfo.UsedPercent } - - latestSystemStatus.Store(status) + return status } // GetSystemStatus 获取当前系统状态 func GetSystemStatus() SystemStatus { return latestSystemStatus.Load().(SystemStatus) } + +// CheckSystemOverload 返回哨兵应据以熔断的过载项。监控关闭、采样缺失或已超过 +// systemStatusMaxAge 时返回 nil。过载标记来自连续采样，阈值按当前配置复核， +// 运行时关闭或调高阈值后立即放行，不必等待下一次采样。 +func CheckSystemOverload(status SystemStatus, config PerformanceMonitorConfig, now time.Time) *SystemOverload { + if !config.Enabled || status.SampledAt.IsZero() || now.Sub(status.SampledAt) > systemStatusMaxAge { + return nil + } + if status.CPUOverloaded && exceedsThreshold(status.CPUUsage, config.CPUThreshold) { + return &SystemOverload{Code: "system_cpu_overloaded", Resource: "cpu", Usage: status.CPUUsage, Threshold: config.CPUThreshold} + } + if status.MemoryOverloaded && exceedsThreshold(status.MemoryUsage, config.MemoryThreshold) { + return &SystemOverload{Code: "system_memory_overloaded", Resource: "memory", Usage: status.MemoryUsage, Threshold: config.MemoryThreshold} + } + if status.DiskOverloaded && exceedsThreshold(status.DiskUsage, config.DiskThreshold) { + return &SystemOverload{Code: "system_disk_overloaded", Resource: "disk", Usage: status.DiskUsage, Threshold: config.DiskThreshold} + } + return nil +} diff --git a/common/system_monitor_test.go b/common/system_monitor_test.go new file mode 100644 index 0000000000..f93dd354bd --- /dev/null +++ b/common/system_monitor_test.go @@ -0,0 +1,280 @@ +package common + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestContainerUsage 在临时目录中构造 cgroup 文件，避免依赖运行环境的真实 cgroup。 +func newTestContainerUsage(t *testing.T, schedulableCPUs float64, files map[string]string) *containerUsage { + t.Helper() + root := t.TempDir() + for name, content := range files { + writeTestCgroupFile(t, filepath.Join(root, name), content) + } + return &containerUsage{cgroupRoot: root, schedulableCPUs: schedulableCPUs} +} + +func writeTestCgroupFile(t *testing.T, path string, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func TestContainerCPUSaturation(t *testing.T) { + tests := []struct { + name string + schedulableCPUs float64 + files map[string]string + usageFile string + usageAfter string + want float64 + }{ + { + name: "cgroup v2 quota", + schedulableCPUs: 4, + files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\nuser_usec 900000\n"}, + usageFile: "cpu.stat", + usageAfter: "usage_usec 6000000\n", // 5 秒用 1 核，配额 2 核 + want: 50, + }, + { + name: "fractional quota", + schedulableCPUs: 4, + files: map[string]string{"cpu.max": "50000 100000\n", "cpu.stat": "usage_usec 1000000\n"}, + usageFile: "cpu.stat", + usageAfter: "usage_usec 3500000\n", // 5 秒用 0.5 核，配额 0.5 核 + want: 100, + }, + { + name: "quota above schedulable CPUs", + schedulableCPUs: 1, + files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\n"}, + usageFile: "cpu.stat", + usageAfter: "usage_usec 6000000\n", // cpuset 只有 1 核，已打满 + want: 100, + }, + { + name: "cgroup v1 quota", + schedulableCPUs: 4, + files: map[string]string{ + "cpu/cpu.cfs_quota_us": "200000\n", + "cpu/cpu.cfs_period_us": "100000\n", + "cpuacct/cpuacct.usage": "1000000000\n", + }, + usageFile: "cpuacct/cpuacct.usage", + usageAfter: "11000000000\n", // 纳秒：5 秒用 2 核 + want: 100, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := newTestContainerUsage(t, tt.schedulableCPUs, tt.files) + base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC) + _, ok := usage.cpuSaturation(base) + assert.False(t, ok, "首次采样只建立基线") + + writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, tt.usageFile), tt.usageAfter) + got, ok := usage.cpuSaturation(base.Add(5 * time.Second)) + require.True(t, ok) + assert.InDelta(t, tt.want, got, 0.01) + }) + } +} + +func TestContainerCPUSaturationFallsBackWithoutQuota(t *testing.T) { + tests := []struct { + name string + files map[string]string + }{ + {name: "cgroup v2 unlimited", files: map[string]string{"cpu.max": "max 100000\n", "cpu.stat": "usage_usec 1000000\n"}}, + {name: "cgroup v1 unlimited", files: map[string]string{"cpu/cpu.cfs_quota_us": "-1\n", "cpu/cpu.cfs_period_us": "100000\n", "cpuacct/cpuacct.usage": "1000\n"}}, + {name: "no cgroup files"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := newTestContainerUsage(t, 4, tt.files) + base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC) + usage.cpuSaturation(base) + _, ok := usage.cpuSaturation(base.Add(5 * time.Second)) + assert.False(t, ok, "没有配额时交由调用方回退宿主机口径") + }) + } +} + +func TestContainerMemoryUsageUsesWorkingSet(t *testing.T) { + tests := []struct { + name string + files map[string]string + want float64 + }{ + { + name: "cgroup v2 excludes inactive file cache", + files: map[string]string{ + "memory.current": "266338304\n", + "memory.max": "268435456\n", + "memory.stat": "anon 4435968\nfile 253362176\ninactive_file 251662336\n", + }, + want: 5.4672, + }, + { + name: "cgroup v2 high working set keeps protection", + files: map[string]string{ + "memory.current": "255852544\n", + "memory.max": "268435456\n", + "memory.stat": "anon 254803968\ninactive_file 1048576\n", + }, + want: 94.9219, + }, + { + name: "cgroup v1 uses hierarchical inactive file cache", + files: map[string]string{ + "memory/memory.usage_in_bytes": "805306368\n", + "memory/memory.limit_in_bytes": "1610612736\n", + "memory/memory.stat": "inactive_file 999\ntotal_inactive_file 268435456\n", + }, + want: 33.3333, + }, + { + name: "cache larger than usage does not underflow", + files: map[string]string{ + "memory.current": "1000\n", + "memory.max": "268435456\n", + "memory.stat": "inactive_file 2000\n", + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := newTestContainerUsage(t, 4, tt.files).memoryUsage() + require.True(t, ok) + assert.InDelta(t, tt.want, got, 0.001) + }) + } +} + +func TestContainerMemoryUsageFallsBackWhenUnavailable(t *testing.T) { + limited := map[string]string{"memory.current": "266338304\n", "memory.max": "268435456\n"} + tests := []struct { + name string + files map[string]string + stat string + }{ + {name: "cgroup v2 unlimited", files: map[string]string{"memory.current": "1000\n", "memory.max": "max\n"}, stat: "inactive_file 0\n"}, + {name: "cgroup v1 unlimited", files: map[string]string{ + "memory/memory.usage_in_bytes": "1000\n", + "memory/memory.limit_in_bytes": "9223372036854771712\n", + "memory/memory.stat": "total_inactive_file 0\n", + }}, + {name: "memory.stat missing", files: limited}, + {name: "inactive file field missing", files: limited, stat: "anon 4435968\n"}, + {name: "inactive file field corrupt", files: limited, stat: "inactive_file abc\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := newTestContainerUsage(t, 4, tt.files) + if tt.stat != "" { + writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, "memory.stat"), tt.stat) + } + _, ok := usage.memoryUsage() + assert.False(t, ok, "拿不到可信工作集时交由调用方回退宿主机口径") + }) + } +} + +func TestSystemOverloadRequiresConsecutiveSamples(t *testing.T) { + config := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95} + now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC) + cpuSamples := []struct { + usage float64 + wantOverload bool + }{ + {usage: 95, wantOverload: false}, + {usage: 95, wantOverload: false}, + {usage: 95, wantOverload: true}, + {usage: 95, wantOverload: true}, + {usage: 50, wantOverload: false}, // 回落即恢复 + {usage: 95, wantOverload: false}, // 重新累计 + {usage: 95, wantOverload: false}, + {usage: 95, wantOverload: true}, + } + + var detector overloadDetector + for i, sample := range cpuSamples { + status := SystemStatus{CPUUsage: sample.usage, SampledAt: now} + detector.mark(&status, config) + overload := CheckSystemOverload(status, config, now) + if !sample.wantOverload { + assert.Nil(t, overload, "sample %d", i+1) + continue + } + require.NotNil(t, overload, "sample %d", i+1) + assert.Equal(t, "system_cpu_overloaded", overload.Code) + } +} + +func TestCheckSystemOverload(t *testing.T) { + now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC) + enabled := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95} + cpuOverloaded := SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now} + tests := []struct { + name string + status SystemStatus + config PerformanceMonitorConfig + wantCode string + wantMessage string + }{ + { + name: "fresh cpu overload", + status: cpuOverloaded, + config: enabled, + wantCode: "system_cpu_overloaded", + wantMessage: "system cpu overloaded (current: 99.5%, threshold: 90%)", + }, + { + name: "fresh memory overload", + status: SystemStatus{MemoryUsage: 96.2, MemoryOverloaded: true, SampledAt: now}, + config: enabled, + wantCode: "system_memory_overloaded", + wantMessage: "system memory overloaded (current: 96.2%, threshold: 90%)", + }, + { + name: "fresh disk overload", + status: SystemStatus{DiskUsage: 97, DiskOverloaded: true, SampledAt: now}, + config: enabled, + wantCode: "system_disk_overloaded", + wantMessage: "system disk overloaded (current: 97.0%, threshold: 95%)", + }, + { + name: "lowered threshold reports the active threshold", + status: cpuOverloaded, + config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 80}, + wantCode: "system_cpu_overloaded", + wantMessage: "system cpu overloaded (current: 99.5%, threshold: 80%)", + }, + {name: "monitor disabled", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: false, CPUThreshold: 90}}, + {name: "cpu threshold disabled at runtime", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 0}}, + {name: "cpu threshold raised above usage", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 100}}, + {name: "stale sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now.Add(-31 * time.Second)}, config: enabled}, + {name: "missing sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true}, config: enabled}, + {name: "usage above threshold without consecutive samples", status: SystemStatus{CPUUsage: 99.5, SampledAt: now}, config: enabled}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + overload := CheckSystemOverload(tt.status, tt.config, now) + if tt.wantCode == "" { + assert.Nil(t, overload) + return + } + require.NotNil(t, overload) + assert.Equal(t, tt.wantCode, overload.Code) + assert.Equal(t, tt.wantMessage, overload.Error()) + }) + } +} diff --git a/middleware/performance.go b/middleware/performance.go index 9cef7f4d15..7d2fa5b1fa 100644 --- a/middleware/performance.go +++ b/middleware/performance.go @@ -1,9 +1,9 @@ package middleware import ( - "fmt" "net/http" "strings" + "time" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/relaykit/types" @@ -39,33 +39,11 @@ func SystemPerformanceCheck() gin.HandlerFunc { // checkSystemPerformance 检查系统性能是否超过阈值 func checkSystemPerformance() *types.NewAPIError { - config := common.GetPerformanceMonitorConfig() - if !config.Enabled { + // 过载标记在采样侧完成：CPU / 内存按容器 cgroup 配额计算，并要求连续多次 + // 超阈值；判定时再按当前阈值复核，并忽略过期采样。 + overload := common.CheckSystemOverload(common.GetSystemStatus(), common.GetPerformanceMonitorConfig(), time.Now()) + if overload == nil { return nil } - - status := common.GetSystemStatus() - - // 检查 CPU - if config.CPUThreshold > 0 && int(status.CPUUsage) > config.CPUThreshold { - return types.NewErrorWithStatusCode( - fmt.Errorf("system cpu overloaded (current: %.1f%%, threshold: %d%%)", status.CPUUsage, config.CPUThreshold), - "system_cpu_overloaded", http.StatusServiceUnavailable) - } - - // 检查内存 - if config.MemoryThreshold > 0 && int(status.MemoryUsage) > config.MemoryThreshold { - return types.NewErrorWithStatusCode( - fmt.Errorf("system memory overloaded (current: %.1f%%, threshold: %d%%)", status.MemoryUsage, config.MemoryThreshold), - "system_memory_overloaded", http.StatusServiceUnavailable) - } - - // 检查磁盘 - if config.DiskThreshold > 0 && int(status.DiskUsage) > config.DiskThreshold { - return types.NewErrorWithStatusCode( - fmt.Errorf("system disk overloaded (current: %.1f%%, threshold: %d%%)", status.DiskUsage, config.DiskThreshold), - "system_disk_overloaded", http.StatusServiceUnavailable) - } - - return nil + return types.NewErrorWithStatusCode(overload, types.ErrorCode(overload.Code), http.StatusServiceUnavailable) }