fix(middleware): 过载哨兵改用容器口径并加入连续性与新鲜度判定 (#7317)

* fix(middleware): 过载哨兵改用容器口径并加入连续性与新鲜度判定

checkSystemPerformance 依赖 common.SystemStatus,而 updateSystemStatus 用
cpu.Percent(0,false) 与 mem.VirtualMemory() 采样,两者都是宿主机口径
(/proc/stat、/proc/meminfo),不具备 cgroup 感知。容器化部署下度量的是
宿主机而非本进程:容器配额 2 核时 /proc/stat 仍暴露宿主机全部核心,同节点
其它工作负载会把该值推高,导致正常 relay 流量被 503 system_cpu_overloaded
拒绝。同时该判定对单次采样即时生效,瞬时尖峰即可熔断。

本次调整:
- 新增 common/container_usage.go:优先按容器 cgroup 配额计算自身 CPU 与内存
  占用(v2 cpu.max/cpu.stat/memory.max/memory.current,兼容 v1),容器口径
  不可用时回退宿主机口径;
- 过载判定要求连续 3 次采样(约 15 秒)超阈值,任一采样回落即恢复;
- 采样超过 30 秒未刷新时哨兵不再熔断,避免状态陈旧误伤流量;
- 中间件改为消费采样结果中的过载标记,错误码与 503 语义不变。

验证:go build ./...、go test ./common/... ./middleware/... 通过;新增
容器口径与判定行为单测。

* fix(monitor): correct container CPU and memory overload detection

---------

Co-authored-by: seefs001 <i@seefs.me>
This commit is contained in:
Alex Xiangandseefs001 authored and GitHub committed 2026-10-16 22:30:06 +08:00
1 parent d032204733
commit 09109ec315
4 files changed
+556 -45

No files matched your search

+174
View File
@@ -0,0 +1,174 @@
package common
import (
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
)
// 容器自身资源采样。
//
// 历史实现直接读取宿主机口径(/proc/stat 与 /proc/meminfo),把同节点其它
// 工作负载的占用也算成本进程过载,导致正常的 relay 流量被 503 熔断。这里
// 改为优先读取当前容器的 cgroup 配额与用量,只有在拿不到 cgroup 数据时才由
// 调用方回退到宿主机口径。
const (
defaultCgroupRoot = "/sys/fs/cgroup"
// cgroup v1 用接近 int64 上限的值表示"未限制"。
unlimitedMemoryBytes = uint64(1) << 60
)
// containerUsage 记录上一次采样点,用来计算区间内的 CPU 占用。
type containerUsage struct {
cgroupRoot string
// schedulableCPUs 是本进程可调度的逻辑 CPU 数(Linux 上含 cpuset/affinity 限制)。
schedulableCPUs float64
cpuLastUsec uint64
cpuLastSampled time.Time
}
func newContainerUsage() *containerUsage {
// runtime.NumCPU 在 Linux 上于进程启动时按 sched_getaffinity 计数,不受 GOMAXPROCS 影响。
return &containerUsage{cgroupRoot: defaultCgroupRoot, schedulableCPUs: float64(runtime.NumCPU())}
}
// cpuSaturation 返回相对容器可用 CPU 的使用率(百分比)。
// 可用 CPU 取 cgroup 配额与可调度 CPU 的较小者:配额大于 cpuset 或节点核数时,
// 按配额计算会让容器打满也无法触发保护。
// ok=false 表示当前环境没有可用的 cgroup CPU 数据,调用方应回退到宿主机口径。
func (u *containerUsage) cpuSaturation(now time.Time) (float64, bool) {
limitCores, ok := u.readCgroupCPUQuotaCores()
if !ok {
return 0, false
}
if u.schedulableCPUs > 0 {
limitCores = min(limitCores, u.schedulableCPUs)
}
usageUsec, ok := u.readCgroupCPUUsageUsec()
if !ok {
return 0, false
}
if u.cpuLastSampled.IsZero() || usageUsec < u.cpuLastUsec || !now.After(u.cpuLastSampled) {
u.cpuLastUsec = usageUsec
u.cpuLastSampled = now
return 0, false
}
elapsed := now.Sub(u.cpuLastSampled)
if elapsed < time.Second {
return 0, false
}
deltaUsec := usageUsec - u.cpuLastUsec
u.cpuLastUsec = usageUsec
u.cpuLastSampled = now
usedCores := float64(deltaUsec) / 1e6 / elapsed.Seconds()
return usedCores / limitCores * 100, true
}
// memoryUsage 返回容器内存工作集相对上限的使用率(百分比)。
// 工作集 = 用量 - 非活跃文件缓存,与 docker stats、kubelet 一致:用量包含可回收的
// 页缓存,写日志或文件后会长期贴近上限,不代表内存压力。
// ok=false 时调用方回退宿主机口径(同样不含缓存):内存未限制、文件不可读、
// 或 memory.stat 缺少非活跃缓存字段。宁可回退旧口径,也不把含缓存的用量当作压力。
func (u *containerUsage) memoryUsage() (float64, bool) {
usagePath := filepath.Join(u.cgroupRoot, "memory.current")
limitPath := filepath.Join(u.cgroupRoot, "memory.max")
statPath := filepath.Join(u.cgroupRoot, "memory.stat")
inactiveFileKey := "inactive_file"
if _, err := os.Stat(usagePath); err != nil {
v1Dir := filepath.Join(u.cgroupRoot, "memory")
usagePath = filepath.Join(v1Dir, "memory.usage_in_bytes")
limitPath = filepath.Join(v1Dir, "memory.limit_in_bytes")
statPath = filepath.Join(v1Dir, "memory.stat")
inactiveFileKey = "total_inactive_file"
}
// cgroup v2 未限制时 memory.max 为 "max",解析失败即视为不可用。
limit, ok := readCgroupUint(limitPath)
if !ok || limit == 0 || limit >= unlimitedMemoryBytes {
return 0, false
}
usage, ok := readCgroupUint(usagePath)
if !ok {
return 0, false
}
inactiveFile, ok := readCgroupStatValue(statPath, inactiveFileKey)
if !ok {
return 0, false
}
// 用量与 memory.stat 不是同一时刻的快照,缓存可能短暂大于用量,按 0 处理防止下溢。
workingSet := usage - min(inactiveFile, usage)
return float64(workingSet) / float64(limit) * 100, true
}
// readCgroupCPUQuotaCores 返回 cgroup CPU 配额(核)。ok=false 表示未限制或不可读。
func (u *containerUsage) readCgroupCPUQuotaCores() (float64, bool) {
if raw, err := os.ReadFile(filepath.Join(u.cgroupRoot, "cpu.max")); err == nil {
quotaText, periodText, found := strings.Cut(strings.TrimSpace(string(raw)), " ")
if !found || quotaText == "max" {
return 0, false
}
quota, err := strconv.ParseUint(quotaText, 10, 64)
if err != nil || quota == 0 {
return 0, false
}
period, err := strconv.ParseUint(strings.TrimSpace(periodText), 10, 64)
if err != nil || period == 0 {
return 0, false
}
return float64(quota) / float64(period), true
}
// cgroup v1 未限制时 cfs_quota_us 为 -1,解析失败即视为不可用。
v1Dir := filepath.Join(u.cgroupRoot, "cpu")
quota, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_quota_us"))
if !ok || quota == 0 {
return 0, false
}
period, ok := readCgroupUint(filepath.Join(v1Dir, "cpu.cfs_period_us"))
if !ok || period == 0 {
return 0, false
}
return float64(quota) / float64(period), true
}
func (u *containerUsage) readCgroupCPUUsageUsec() (uint64, bool) {
statPath := filepath.Join(u.cgroupRoot, "cpu.stat")
if _, err := os.Stat(statPath); err == nil {
return readCgroupStatValue(statPath, "usage_usec")
}
// cgroup v1 的 cpuacct.usage 单位是纳秒。
nanos, ok := readCgroupUint(filepath.Join(u.cgroupRoot, "cpuacct", "cpuacct.usage"))
return nanos / 1000, ok
}
// readCgroupUint 读取只含一个无符号整数的 cgroup 文件。
func readCgroupUint(path string) (uint64, bool) {
raw, err := os.ReadFile(path)
if err != nil {
return 0, false
}
value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64)
return value, err == nil
}
// readCgroupStatValue 从 "key value" 逐行格式的 cgroup 统计文件(cpu.stat、memory.stat)中读取一项。
func readCgroupStatValue(path string, key string) (uint64, bool) {
raw, err := os.ReadFile(path)
if err != nil {
return 0, false
}
for line := range strings.SplitSeq(string(raw), "\n") {
name, valueText, found := strings.Cut(line, " ")
if !found || name != key {
continue
}
value, err := strconv.ParseUint(strings.TrimSpace(valueText), 10, 64)
return value, err == nil
}
return 0, false
}
+96 -17
View File
@@ -1,6 +1,7 @@
package common
import (
"fmt"
"sync/atomic"
"time"
@@ -25,10 +26,36 @@ type SystemStatus struct {
CPUUsage float64
MemoryUsage float64
DiskUsage float64
SampledAt time.Time
// 过载判定带连续性要求:单一采样尖峰不再直接熔断,需连续多次超阈值。
CPUOverloaded bool
MemoryOverloaded bool
DiskOverloaded bool
}
// SystemOverload 是哨兵据以拒绝请求的过载项,Error 文本带当前值与生效阈值。
type SystemOverload struct {
Code string
Resource string
Usage float64
Threshold int
}
func (o *SystemOverload) Error() string {
return fmt.Sprintf("system %s overloaded (current: %.1f%%, threshold: %d%%)", o.Resource, o.Usage, o.Threshold)
}
var latestSystemStatus atomic.Value
const (
systemMonitorInterval = 5 * time.Second
// 连续 3 次采样(约 15 秒)仍高于阈值才判定过载,
// 任一采样回落到阈值以下即恢复,避免瞬时尖峰打满重试。
overloadConsecutiveSamples = 3
// 采样超过该时长未刷新(监控未运行或被阻塞)时,哨兵不据此熔断。
systemStatusMaxAge = 30 * time.Second
)
func init() {
latestSystemStatus.Store(SystemStatus{})
}
@@ -36,46 +63,98 @@ func init() {
// StartSystemMonitor 启动系统监控
func StartSystemMonitor() {
go func() {
usage := newContainerUsage()
var detector overloadDetector
for {
config := GetPerformanceMonitorConfig()
if !config.Enabled {
time.Sleep(30 * time.Second)
detector = overloadDetector{}
latestSystemStatus.Store(SystemStatus{SampledAt: time.Now()})
time.Sleep(systemMonitorInterval)
continue
}
updateSystemStatus()
time.Sleep(5 * time.Second)
status := collectSystemStatus(usage)
detector.mark(&status, config)
latestSystemStatus.Store(status)
time.Sleep(systemMonitorInterval)
}
}()
}
func updateSystemStatus() {
var status SystemStatus
// overloadDetector 为每项资源累计连续超阈值的采样次数:连续 overloadConsecutiveSamples 次
// 才标记过载,任一次回落立即清零并重新累计。
type overloadDetector struct {
cpuSamples int
memorySamples int
diskSamples int
}
// CPU
// 注意:cpu.Percent(0, false) 返回自上次调用以来的 CPU 使用率
// 如果是第一次调用,可能会返回错误或不准确的值,但在循环中会逐渐正常
percents, err := cpu.Percent(0, false)
if err == nil && len(percents) > 0 {
func (d *overloadDetector) mark(status *SystemStatus, config PerformanceMonitorConfig) {
status.CPUOverloaded = countOverloadSample(&d.cpuSamples, exceedsThreshold(status.CPUUsage, config.CPUThreshold))
status.MemoryOverloaded = countOverloadSample(&d.memorySamples, exceedsThreshold(status.MemoryUsage, config.MemoryThreshold))
status.DiskOverloaded = countOverloadSample(&d.diskSamples, exceedsThreshold(status.DiskUsage, config.DiskThreshold))
}
func countOverloadSample(samples *int, over bool) bool {
if !over {
*samples = 0
return false
}
*samples = min(*samples+1, overloadConsecutiveSamples)
return *samples == overloadConsecutiveSamples
}
// exceedsThreshold 判断采样值是否超过阈值;阈值 <= 0 表示不检查该项。
func exceedsThreshold(usage float64, threshold int) bool {
return threshold > 0 && int(usage) > threshold
}
func collectSystemStatus(usage *containerUsage) SystemStatus {
now := time.Now()
status := SystemStatus{SampledAt: now}
// CPU / 内存优先按容器 cgroup 配额计算;容器口径不可用时回退宿主机口径,
// 保持对未限制资源或被挂载为宿主 cgroup 的环境的兼容。
if saturation, ok := usage.cpuSaturation(now); ok {
status.CPUUsage = saturation
} else if percents, err := cpu.Percent(0, false); err == nil && len(percents) > 0 {
status.CPUUsage = percents[0]
}
// Memory
memInfo, err := mem.VirtualMemory()
if err == nil {
if containerMemory, ok := usage.memoryUsage(); ok {
status.MemoryUsage = containerMemory
} else if memInfo, err := mem.VirtualMemory(); err == nil {
status.MemoryUsage = memInfo.UsedPercent
}
// Disk
diskInfo := GetDiskSpaceInfo()
if diskInfo.Total > 0 {
status.DiskUsage = diskInfo.UsedPercent
}
latestSystemStatus.Store(status)
return status
}
// GetSystemStatus 获取当前系统状态
func GetSystemStatus() SystemStatus {
return latestSystemStatus.Load().(SystemStatus)
}
// CheckSystemOverload 返回哨兵应据以熔断的过载项。监控关闭、采样缺失或已超过
// systemStatusMaxAge 时返回 nil。过载标记来自连续采样,阈值按当前配置复核,
// 运行时关闭或调高阈值后立即放行,不必等待下一次采样。
func CheckSystemOverload(status SystemStatus, config PerformanceMonitorConfig, now time.Time) *SystemOverload {
if !config.Enabled || status.SampledAt.IsZero() || now.Sub(status.SampledAt) > systemStatusMaxAge {
return nil
}
if status.CPUOverloaded && exceedsThreshold(status.CPUUsage, config.CPUThreshold) {
return &SystemOverload{Code: "system_cpu_overloaded", Resource: "cpu", Usage: status.CPUUsage, Threshold: config.CPUThreshold}
}
if status.MemoryOverloaded && exceedsThreshold(status.MemoryUsage, config.MemoryThreshold) {
return &SystemOverload{Code: "system_memory_overloaded", Resource: "memory", Usage: status.MemoryUsage, Threshold: config.MemoryThreshold}
}
if status.DiskOverloaded && exceedsThreshold(status.DiskUsage, config.DiskThreshold) {
return &SystemOverload{Code: "system_disk_overloaded", Resource: "disk", Usage: status.DiskUsage, Threshold: config.DiskThreshold}
}
return nil
}
+280
View File
@@ -0,0 +1,280 @@
package common
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newTestContainerUsage 在临时目录中构造 cgroup 文件,避免依赖运行环境的真实 cgroup。
func newTestContainerUsage(t *testing.T, schedulableCPUs float64, files map[string]string) *containerUsage {
t.Helper()
root := t.TempDir()
for name, content := range files {
writeTestCgroupFile(t, filepath.Join(root, name), content)
}
return &containerUsage{cgroupRoot: root, schedulableCPUs: schedulableCPUs}
}
func writeTestCgroupFile(t *testing.T, path string, content string) {
t.Helper()
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
}
func TestContainerCPUSaturation(t *testing.T) {
tests := []struct {
name string
schedulableCPUs float64
files map[string]string
usageFile string
usageAfter string
want float64
}{
{
name: "cgroup v2 quota",
schedulableCPUs: 4,
files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\nuser_usec 900000\n"},
usageFile: "cpu.stat",
usageAfter: "usage_usec 6000000\n", // 5 秒用 1 核,配额 2 核
want: 50,
},
{
name: "fractional quota",
schedulableCPUs: 4,
files: map[string]string{"cpu.max": "50000 100000\n", "cpu.stat": "usage_usec 1000000\n"},
usageFile: "cpu.stat",
usageAfter: "usage_usec 3500000\n", // 5 秒用 0.5 核,配额 0.5 核
want: 100,
},
{
name: "quota above schedulable CPUs",
schedulableCPUs: 1,
files: map[string]string{"cpu.max": "200000 100000\n", "cpu.stat": "usage_usec 1000000\n"},
usageFile: "cpu.stat",
usageAfter: "usage_usec 6000000\n", // cpuset 只有 1 核,已打满
want: 100,
},
{
name: "cgroup v1 quota",
schedulableCPUs: 4,
files: map[string]string{
"cpu/cpu.cfs_quota_us": "200000\n",
"cpu/cpu.cfs_period_us": "100000\n",
"cpuacct/cpuacct.usage": "1000000000\n",
},
usageFile: "cpuacct/cpuacct.usage",
usageAfter: "11000000000\n", // 纳秒:5 秒用 2 核
want: 100,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
usage := newTestContainerUsage(t, tt.schedulableCPUs, tt.files)
base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
_, ok := usage.cpuSaturation(base)
assert.False(t, ok, "首次采样只建立基线")
writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, tt.usageFile), tt.usageAfter)
got, ok := usage.cpuSaturation(base.Add(5 * time.Second))
require.True(t, ok)
assert.InDelta(t, tt.want, got, 0.01)
})
}
}
func TestContainerCPUSaturationFallsBackWithoutQuota(t *testing.T) {
tests := []struct {
name string
files map[string]string
}{
{name: "cgroup v2 unlimited", files: map[string]string{"cpu.max": "max 100000\n", "cpu.stat": "usage_usec 1000000\n"}},
{name: "cgroup v1 unlimited", files: map[string]string{"cpu/cpu.cfs_quota_us": "-1\n", "cpu/cpu.cfs_period_us": "100000\n", "cpuacct/cpuacct.usage": "1000\n"}},
{name: "no cgroup files"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
usage := newTestContainerUsage(t, 4, tt.files)
base := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
usage.cpuSaturation(base)
_, ok := usage.cpuSaturation(base.Add(5 * time.Second))
assert.False(t, ok, "没有配额时交由调用方回退宿主机口径")
})
}
}
func TestContainerMemoryUsageUsesWorkingSet(t *testing.T) {
tests := []struct {
name string
files map[string]string
want float64
}{
{
name: "cgroup v2 excludes inactive file cache",
files: map[string]string{
"memory.current": "266338304\n",
"memory.max": "268435456\n",
"memory.stat": "anon 4435968\nfile 253362176\ninactive_file 251662336\n",
},
want: 5.4672,
},
{
name: "cgroup v2 high working set keeps protection",
files: map[string]string{
"memory.current": "255852544\n",
"memory.max": "268435456\n",
"memory.stat": "anon 254803968\ninactive_file 1048576\n",
},
want: 94.9219,
},
{
name: "cgroup v1 uses hierarchical inactive file cache",
files: map[string]string{
"memory/memory.usage_in_bytes": "805306368\n",
"memory/memory.limit_in_bytes": "1610612736\n",
"memory/memory.stat": "inactive_file 999\ntotal_inactive_file 268435456\n",
},
want: 33.3333,
},
{
name: "cache larger than usage does not underflow",
files: map[string]string{
"memory.current": "1000\n",
"memory.max": "268435456\n",
"memory.stat": "inactive_file 2000\n",
},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := newTestContainerUsage(t, 4, tt.files).memoryUsage()
require.True(t, ok)
assert.InDelta(t, tt.want, got, 0.001)
})
}
}
func TestContainerMemoryUsageFallsBackWhenUnavailable(t *testing.T) {
limited := map[string]string{"memory.current": "266338304\n", "memory.max": "268435456\n"}
tests := []struct {
name string
files map[string]string
stat string
}{
{name: "cgroup v2 unlimited", files: map[string]string{"memory.current": "1000\n", "memory.max": "max\n"}, stat: "inactive_file 0\n"},
{name: "cgroup v1 unlimited", files: map[string]string{
"memory/memory.usage_in_bytes": "1000\n",
"memory/memory.limit_in_bytes": "9223372036854771712\n",
"memory/memory.stat": "total_inactive_file 0\n",
}},
{name: "memory.stat missing", files: limited},
{name: "inactive file field missing", files: limited, stat: "anon 4435968\n"},
{name: "inactive file field corrupt", files: limited, stat: "inactive_file abc\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
usage := newTestContainerUsage(t, 4, tt.files)
if tt.stat != "" {
writeTestCgroupFile(t, filepath.Join(usage.cgroupRoot, "memory.stat"), tt.stat)
}
_, ok := usage.memoryUsage()
assert.False(t, ok, "拿不到可信工作集时交由调用方回退宿主机口径")
})
}
}
func TestSystemOverloadRequiresConsecutiveSamples(t *testing.T) {
config := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95}
now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
cpuSamples := []struct {
usage float64
wantOverload bool
}{
{usage: 95, wantOverload: false},
{usage: 95, wantOverload: false},
{usage: 95, wantOverload: true},
{usage: 95, wantOverload: true},
{usage: 50, wantOverload: false}, // 回落即恢复
{usage: 95, wantOverload: false}, // 重新累计
{usage: 95, wantOverload: false},
{usage: 95, wantOverload: true},
}
var detector overloadDetector
for i, sample := range cpuSamples {
status := SystemStatus{CPUUsage: sample.usage, SampledAt: now}
detector.mark(&status, config)
overload := CheckSystemOverload(status, config, now)
if !sample.wantOverload {
assert.Nil(t, overload, "sample %d", i+1)
continue
}
require.NotNil(t, overload, "sample %d", i+1)
assert.Equal(t, "system_cpu_overloaded", overload.Code)
}
}
func TestCheckSystemOverload(t *testing.T) {
now := time.Date(2026, 9, 11, 3, 0, 0, 0, time.UTC)
enabled := PerformanceMonitorConfig{Enabled: true, CPUThreshold: 90, MemoryThreshold: 90, DiskThreshold: 95}
cpuOverloaded := SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now}
tests := []struct {
name string
status SystemStatus
config PerformanceMonitorConfig
wantCode string
wantMessage string
}{
{
name: "fresh cpu overload",
status: cpuOverloaded,
config: enabled,
wantCode: "system_cpu_overloaded",
wantMessage: "system cpu overloaded (current: 99.5%, threshold: 90%)",
},
{
name: "fresh memory overload",
status: SystemStatus{MemoryUsage: 96.2, MemoryOverloaded: true, SampledAt: now},
config: enabled,
wantCode: "system_memory_overloaded",
wantMessage: "system memory overloaded (current: 96.2%, threshold: 90%)",
},
{
name: "fresh disk overload",
status: SystemStatus{DiskUsage: 97, DiskOverloaded: true, SampledAt: now},
config: enabled,
wantCode: "system_disk_overloaded",
wantMessage: "system disk overloaded (current: 97.0%, threshold: 95%)",
},
{
name: "lowered threshold reports the active threshold",
status: cpuOverloaded,
config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 80},
wantCode: "system_cpu_overloaded",
wantMessage: "system cpu overloaded (current: 99.5%, threshold: 80%)",
},
{name: "monitor disabled", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: false, CPUThreshold: 90}},
{name: "cpu threshold disabled at runtime", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 0}},
{name: "cpu threshold raised above usage", status: cpuOverloaded, config: PerformanceMonitorConfig{Enabled: true, CPUThreshold: 100}},
{name: "stale sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true, SampledAt: now.Add(-31 * time.Second)}, config: enabled},
{name: "missing sample", status: SystemStatus{CPUUsage: 99.5, CPUOverloaded: true}, config: enabled},
{name: "usage above threshold without consecutive samples", status: SystemStatus{CPUUsage: 99.5, SampledAt: now}, config: enabled},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
overload := CheckSystemOverload(tt.status, tt.config, now)
if tt.wantCode == "" {
assert.Nil(t, overload)
return
}
require.NotNil(t, overload)
assert.Equal(t, tt.wantCode, overload.Code)
assert.Equal(t, tt.wantMessage, overload.Error())
})
}
}
+6 -28
View File
@@ -1,9 +1,9 @@
package middleware
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/relaykit/types"
@@ -39,33 +39,11 @@ func SystemPerformanceCheck() gin.HandlerFunc {
// checkSystemPerformance 检查系统性能是否超过阈值
func checkSystemPerformance() *types.NewAPIError {
config := common.GetPerformanceMonitorConfig()
if !config.Enabled {
// 过载标记在采样侧完成:CPU / 内存按容器 cgroup 配额计算,并要求连续多次
// 超阈值;判定时再按当前阈值复核,并忽略过期采样。
overload := common.CheckSystemOverload(common.GetSystemStatus(), common.GetPerformanceMonitorConfig(), time.Now())
if overload == nil {
return nil
}
status := common.GetSystemStatus()
// 检查 CPU
if config.CPUThreshold > 0 && int(status.CPUUsage) > config.CPUThreshold {
return types.NewErrorWithStatusCode(
fmt.Errorf("system cpu overloaded (current: %.1f%%, threshold: %d%%)", status.CPUUsage, config.CPUThreshold),
"system_cpu_overloaded", http.StatusServiceUnavailable)
}
// 检查内存
if config.MemoryThreshold > 0 && int(status.MemoryUsage) > config.MemoryThreshold {
return types.NewErrorWithStatusCode(
fmt.Errorf("system memory overloaded (current: %.1f%%, threshold: %d%%)", status.MemoryUsage, config.MemoryThreshold),
"system_memory_overloaded", http.StatusServiceUnavailable)
}
// 检查磁盘
if config.DiskThreshold > 0 && int(status.DiskUsage) > config.DiskThreshold {
return types.NewErrorWithStatusCode(
fmt.Errorf("system disk overloaded (current: %.1f%%, threshold: %d%%)", status.DiskUsage, config.DiskThreshold),
"system_disk_overloaded", http.StatusServiceUnavailable)
}
return nil
return types.NewErrorWithStatusCode(overload, types.ErrorCode(overload.Code), http.StatusServiceUnavailable)
}