opsx
572efc89a4
- CollectorService 安装/卸载按采集器互斥去重(重复触发不再并发 docker rm/run 互踩);删除后卸载失败留日志 - ApplyStagedBackup 替换文件后立即 Reload 内存态,消除「重启前旧内存落盘覆盖恢复文件」竞态;暂存文件名升级 lokistack.json(兼容旧名) - GetUsage/Validate 全链路 context 贯穿:请求取消即中止 SSH/k8s 查询(异步调用点显式 Background)
488 lines
16 KiB
Go
488 lines
16 KiB
Go
package services
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||
|
||
"lokistack/models"
|
||
)
|
||
|
||
// ResourceUsage 是页面上「资源」弹框的数据。
|
||
//
|
||
// 三种部署模式都能采集：
|
||
// - k8s-yaml : StatefulSet resources + metrics-server（实时用量）+ 可选 Prometheus（峰值/限流）
|
||
// - host-docker : docker stats / docker inspect（容器限额未设则按宿主机内存做参考）
|
||
// - host-binary : ps 采集进程 CPU%/RSS（无 cgroup 限额）
|
||
//
|
||
// 权限兼容：集群 token 常是最小权限，可能没有 metrics.k8s.io。
|
||
// 这时**不报错**，照样返回 limits/就绪/重启，并把原因写进 MetricsNote 让页面提示。
|
||
type ResourceUsage struct {
|
||
Mode string `json:"mode"`
|
||
Source string `json:"source"`
|
||
Namespace string `json:"namespace,omitempty"`
|
||
Pod string `json:"pod,omitempty"`
|
||
Node string `json:"node,omitempty"`
|
||
Ready bool `json:"ready"`
|
||
Restarts int32 `json:"restarts"`
|
||
|
||
CPUCurrentMilli int64 `json:"cpuCurrentMilli"`
|
||
CPURequestMilli int64 `json:"cpuRequestMilli"`
|
||
CPULimitMilli int64 `json:"cpuLimitMilli"`
|
||
|
||
MemCurrentBytes int64 `json:"memCurrentBytes"`
|
||
MemRequestBytes int64 `json:"memRequestBytes"`
|
||
MemLimitBytes int64 `json:"memLimitBytes"`
|
||
|
||
// 限额是否真的配置了（docker 未加 --memory 时为 false，页面不画百分比）
|
||
LimitsConfigured bool `json:"limitsConfigured"`
|
||
|
||
// 可选（需 Prometheus）
|
||
CPUPeakMilli *int64 `json:"cpuPeakMilli,omitempty"`
|
||
MemPeakBytes *int64 `json:"memPeakBytes,omitempty"`
|
||
ThrottlePercent *float64 `json:"throttlePercent,omitempty"`
|
||
|
||
// 采集受限说明（如缺 metrics.k8s.io 权限、未装 metrics-server、docker 无历史数据）
|
||
MetricsNote string `json:"metricsNote,omitempty"`
|
||
|
||
CPUSuggest string `json:"cpuSuggest,omitempty"`
|
||
MemSuggest string `json:"memSuggest,omitempty"`
|
||
Summary string `json:"summary"`
|
||
}
|
||
|
||
type podMetricsResponse struct {
|
||
Containers []struct {
|
||
Name string `json:"name"`
|
||
Usage struct {
|
||
CPU string `json:"cpu"`
|
||
Memory string `json:"memory"`
|
||
} `json:"usage"`
|
||
} `json:"containers"`
|
||
}
|
||
|
||
// GetUsage 采集某个 Loki 部署的资源用量（按部署模式分派）。
|
||
func (s *DeploymentService) GetUsage(ctx context.Context, id uint) (*ResourceUsage, error) {
|
||
d, err := s.Get(id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
switch d.Mode {
|
||
case "k8s-yaml":
|
||
return s.k8sUsage(ctx, d)
|
||
case "host-docker":
|
||
return s.dockerUsage(ctx, d)
|
||
case "host-binary":
|
||
return s.processUsage(ctx, d)
|
||
default:
|
||
return nil, fmt.Errorf("不支持的部署模式：%s", d.Mode)
|
||
}
|
||
}
|
||
|
||
// ---------- Kubernetes ----------
|
||
|
||
func (s *DeploymentService) k8sUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||
if d.Cluster == nil {
|
||
return nil, fmt.Errorf("部署未关联 Kubernetes 集群")
|
||
}
|
||
client, err := clusterClient(d.Cluster, 20*time.Second)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||
defer cancel()
|
||
|
||
ns := d.Namespace
|
||
if ns == "" {
|
||
ns = "log"
|
||
}
|
||
usage := &ResourceUsage{Mode: d.Mode, Source: "kubernetes", Namespace: ns, Pod: "loki-0"}
|
||
|
||
// 1) limits/requests 取线上 StatefulSet（以运行态为准，不信部署记录）
|
||
if sts, err := client.AppsV1().StatefulSets(ns).Get(ctx, "loki", metav1.GetOptions{}); err == nil &&
|
||
len(sts.Spec.Template.Spec.Containers) > 0 {
|
||
res := sts.Spec.Template.Spec.Containers[0].Resources
|
||
usage.CPULimitMilli = parseQuantityMilli(res.Limits.Cpu().String())
|
||
usage.CPURequestMilli = parseQuantityMilli(res.Requests.Cpu().String())
|
||
usage.MemLimitBytes = parseQuantityBytes(res.Limits.Memory().String())
|
||
usage.MemRequestBytes = parseQuantityBytes(res.Requests.Memory().String())
|
||
usage.LimitsConfigured = usage.CPULimitMilli > 0 && usage.MemLimitBytes > 0
|
||
}
|
||
|
||
// 2) Pod 状态
|
||
if pod, err := client.CoreV1().Pods(ns).Get(ctx, usage.Pod, metav1.GetOptions{}); err == nil {
|
||
usage.Node = pod.Spec.NodeName
|
||
for _, cs := range pod.Status.ContainerStatuses {
|
||
if cs.Name == "loki" {
|
||
usage.Ready = cs.Ready
|
||
usage.Restarts = cs.RestartCount
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3) 实时用量：metrics-server（最小权限 token 常常没有这个 API 组，要降级而不是报错）
|
||
raw, err := client.Discovery().RESTClient().Get().
|
||
AbsPath("/apis/metrics.k8s.io/v1beta1/namespaces/" + ns + "/pods/" + usage.Pod).
|
||
DoRaw(ctx)
|
||
switch {
|
||
case err != nil:
|
||
usage.MetricsNote = explainMetricsError(err)
|
||
case json.Unmarshal(raw, &podMetricsResponse{}) == nil:
|
||
var pm podMetricsResponse
|
||
if json.Unmarshal(raw, &pm) == nil {
|
||
for _, c := range pm.Containers {
|
||
if c.Name == "loki" {
|
||
usage.CPUCurrentMilli = parseQuantityMilli(c.Usage.CPU)
|
||
usage.MemCurrentBytes = parseQuantityBytes(c.Usage.Memory)
|
||
}
|
||
}
|
||
}
|
||
default:
|
||
usage.MetricsNote = "无法解析 metrics-server 返回"
|
||
}
|
||
|
||
s.applyPrometheus(usage)
|
||
usage.buildHints()
|
||
return usage, nil
|
||
}
|
||
|
||
// explainMetricsError 把 RBAC/安装问题翻译成可执行的提示。
|
||
func explainMetricsError(err error) string {
|
||
msg := strings.ToLower(err.Error())
|
||
switch {
|
||
case strings.Contains(msg, "forbidden") || strings.Contains(msg, "cannot get resource"):
|
||
return "当前集群凭证缺少 metrics.k8s.io 权限，看不到实时用量。需要给该 ServiceAccount 增加：" +
|
||
"apiGroups=[\"metrics.k8s.io\"] resources=[\"pods\"] verbs=[\"get\",\"list\"]（其余限额/就绪数据仍可正常显示）"
|
||
case strings.Contains(msg, "not found") || strings.Contains(msg, "could not find the requested resource"):
|
||
return "集群未安装 metrics-server，无法获取实时 CPU/内存用量（限额与就绪状态仍可显示）"
|
||
default:
|
||
return "读取实时用量失败：" + err.Error()
|
||
}
|
||
}
|
||
|
||
// applyPrometheus 补 24h 峰值与 CPU 限流率（未配置 Prometheus 地址则跳过）。
|
||
func (s *DeploymentService) applyPrometheus(u *ResourceUsage) {
|
||
base := strings.TrimSpace(os.Getenv("LOKI_USAGE_PROMETHEUS_URL"))
|
||
if base == "" || u.Pod == "" {
|
||
if u.MetricsNote == "" {
|
||
u.MetricsNote = "未配置 Prometheus，暂无 24h 峰值与限流率"
|
||
}
|
||
return
|
||
}
|
||
sel := fmt.Sprintf(`pod=%q,namespace=%q,container="loki"`, u.Pod, u.Namespace)
|
||
if v, ok := promQueryValue(base, fmt.Sprintf(`max_over_time(sum(rate(container_cpu_usage_seconds_total{%s}[5m]))[24h:1m])`, sel)); ok {
|
||
milli := int64(v * 1000)
|
||
u.CPUPeakMilli = &milli
|
||
}
|
||
if v, ok := promQueryValue(base, fmt.Sprintf(`max_over_time(sum(container_memory_working_set_bytes{%s})[24h:1m])`, sel)); ok {
|
||
b := int64(v)
|
||
u.MemPeakBytes = &b
|
||
}
|
||
if v, ok := promQueryValue(base, fmt.Sprintf(`100 * sum(rate(container_cpu_cfs_throttled_periods_total{pod=%q,namespace=%q}[15m])) / sum(rate(container_cpu_cfs_periods_total{pod=%q,namespace=%q}[15m]))`, u.Pod, u.Namespace, u.Pod, u.Namespace)); ok {
|
||
u.ThrottlePercent = &v
|
||
}
|
||
}
|
||
|
||
// ---------- host-docker ----------
|
||
|
||
func (s *DeploymentService) dockerUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||
if d.Host == nil {
|
||
return nil, fmt.Errorf("部署未关联主机")
|
||
}
|
||
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("连接部署主机失败：%w", err)
|
||
}
|
||
defer client.Close()
|
||
|
||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||
defer cancel()
|
||
|
||
name := dockerContainerName(d.ID)
|
||
usage := &ResourceUsage{Mode: d.Mode, Source: "docker", Pod: name, Node: d.Host.HostIP}
|
||
|
||
// 容器状态 + 限额（docker run 未指定 --memory/--cpus 时限额为 0 = 不限制）
|
||
inspectFmt := "{{.State.Status}}|{{.RestartCount}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}"
|
||
out, stderr, err := client.Execute(ctx, fmt.Sprintf("docker inspect -f '%s' %s 2>&1", inspectFmt, name))
|
||
if err != nil || strings.Contains(out, "No such object") {
|
||
return nil, fmt.Errorf("容器 %s 不存在或主机无法访问：%s", name, strings.TrimSpace(stderr))
|
||
}
|
||
parts := strings.Split(strings.TrimSpace(out), "|")
|
||
if len(parts) >= 4 {
|
||
usage.Ready = parts[0] == "running"
|
||
if n, err := strconv.Atoi(parts[1]); err == nil {
|
||
usage.Restarts = int32(n)
|
||
}
|
||
if nano, err := strconv.ParseInt(parts[2], 10, 64); err == nil && nano > 0 {
|
||
usage.CPULimitMilli = nano / 1e6
|
||
}
|
||
if mem, err := strconv.ParseInt(parts[3], 10, 64); err == nil && mem > 0 {
|
||
usage.MemLimitBytes = mem
|
||
}
|
||
usage.LimitsConfigured = usage.CPULimitMilli > 0 && usage.MemLimitBytes > 0
|
||
}
|
||
|
||
// 实时用量：docker stats（一次采样）
|
||
statsFmt := "{{.CPUPerc}}|{{.MemUsage}}"
|
||
out, _, err = client.Execute(ctx, fmt.Sprintf("docker stats --no-stream --format '%s' %s 2>/dev/null", statsFmt, name))
|
||
if err == nil {
|
||
if fields := strings.Split(strings.TrimSpace(out), "|"); len(fields) >= 2 {
|
||
usage.CPUCurrentMilli = int64(parsePercent(fields[0]) * 10) // 1% = 10m
|
||
used, limit := parseDockerMemUsage(fields[1])
|
||
usage.MemCurrentBytes = used
|
||
if usage.MemLimitBytes == 0 {
|
||
usage.MemLimitBytes = limit // 未设限额时 docker 报宿主机内存，作为参考上限
|
||
}
|
||
}
|
||
} else {
|
||
usage.MetricsNote = "docker stats 读取失败：" + err.Error()
|
||
}
|
||
if usage.MetricsNote == "" {
|
||
usage.MetricsNote = "Docker 部署无历史峰值；如需 24h 峰值请接入 cAdvisor/Prometheus"
|
||
}
|
||
|
||
usage.buildHints()
|
||
return usage, nil
|
||
}
|
||
|
||
// ---------- host-binary ----------
|
||
|
||
func (s *DeploymentService) processUsage(ctx context.Context, d *models.LokiDeployment) (*ResourceUsage, error) {
|
||
if d.Host == nil {
|
||
return nil, fmt.Errorf("部署未关联主机")
|
||
}
|
||
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("连接部署主机失败：%w", err)
|
||
}
|
||
defer client.Close()
|
||
|
||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||
defer cancel()
|
||
|
||
usage := &ResourceUsage{Mode: d.Mode, Source: "binary", Pod: "loki", Node: d.Host.HostIP}
|
||
out, _, err := client.Execute(ctx, "ps -eo pcpu,rss,comm --no-headers | awk '$3==\"loki\"{print $1\"|\"$2; exit}'")
|
||
if err != nil || strings.TrimSpace(out) == "" {
|
||
return nil, fmt.Errorf("主机上未找到 loki 进程")
|
||
}
|
||
if fields := strings.Split(strings.TrimSpace(out), "|"); len(fields) >= 2 {
|
||
usage.CPUCurrentMilli = int64(parseFloatSafe(fields[0]) * 10) // %CPU → millicores（单核百分比）
|
||
rssKB, _ := strconv.ParseInt(strings.TrimSpace(fields[1]), 10, 64)
|
||
usage.MemCurrentBytes = rssKB * 1024
|
||
usage.Ready = true
|
||
}
|
||
usage.MetricsNote = "裸进程部署没有 cgroup 限额，只能看实时占用（无峰值/限额对比）"
|
||
|
||
usage.buildHints()
|
||
return usage, nil
|
||
}
|
||
|
||
// ---------- 共用：扩容建议 ----------
|
||
|
||
func (u *ResourceUsage) buildHints() {
|
||
cpuRef := u.CPUCurrentMilli
|
||
if u.CPUPeakMilli != nil && *u.CPUPeakMilli > cpuRef {
|
||
cpuRef = *u.CPUPeakMilli
|
||
}
|
||
memRef := u.MemCurrentBytes
|
||
if u.MemPeakBytes != nil && *u.MemPeakBytes > memRef {
|
||
memRef = *u.MemPeakBytes
|
||
}
|
||
|
||
var notes []string
|
||
if u.CPULimitMilli > 0 {
|
||
ratio := float64(cpuRef) / float64(u.CPULimitMilli)
|
||
switch {
|
||
case ratio >= 0.9:
|
||
u.CPUSuggest = fmt.Sprintf("CPU 峰值已达 limit 的 %.0f%%，建议把 CPU limit 提升到 %s 核", ratio*100, formatMilli(nextCPU(u.CPULimitMilli)))
|
||
notes = append(notes, "CPU 打满")
|
||
case ratio >= 0.7:
|
||
u.CPUSuggest = fmt.Sprintf("CPU 峰值占 limit 的 %.0f%%，接近上限，可考虑提前扩容", ratio*100)
|
||
notes = append(notes, "CPU 偏高")
|
||
}
|
||
}
|
||
if u.MemLimitBytes > 0 {
|
||
ratio := float64(memRef) / float64(u.MemLimitBytes)
|
||
switch {
|
||
case ratio >= 0.9:
|
||
u.MemSuggest = fmt.Sprintf("内存峰值已达 limit 的 %.0f%%，有 OOMKilled 风险，建议提升到 %s", ratio*100, formatBytesShort(nextMem(u.MemLimitBytes)))
|
||
notes = append(notes, "内存吃紧")
|
||
case ratio >= 0.75:
|
||
u.MemSuggest = fmt.Sprintf("内存峰值占 limit 的 %.0f%%，接近上限，可考虑提前扩容", ratio*100)
|
||
notes = append(notes, "内存偏高")
|
||
}
|
||
}
|
||
if u.ThrottlePercent != nil && *u.ThrottlePercent >= 10 {
|
||
notes = append(notes, fmt.Sprintf("CPU 限流 %.0f%%", *u.ThrottlePercent))
|
||
}
|
||
if !u.Ready {
|
||
notes = append(notes, "实例未就绪")
|
||
}
|
||
|
||
if len(notes) == 0 {
|
||
if u.LimitsConfigured {
|
||
u.Summary = "资源充足：CPU / 内存均低于 limit 的 70%"
|
||
} else {
|
||
u.Summary = "未配置资源限额（建议设置 CPU / 内存上限，便于判断扩容时机）"
|
||
}
|
||
} else {
|
||
u.Summary = "需要关注：" + strings.Join(notes, "、")
|
||
}
|
||
}
|
||
|
||
// ---------- Prometheus ----------
|
||
|
||
// promQueryValue 查 Prometheus 瞬时值；未配置或失败返回 ok=false（页面降级）。
|
||
func promQueryValue(base, expr string) (float64, bool) {
|
||
client := &http.Client{Timeout: 8 * time.Second}
|
||
u := strings.TrimSuffix(base, "/") + "/api/v1/query?query=" + url.QueryEscape(expr)
|
||
resp, err := client.Get(u)
|
||
if err != nil {
|
||
return 0, false
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode >= 400 {
|
||
return 0, false
|
||
}
|
||
var payload struct {
|
||
Data struct {
|
||
Result []struct {
|
||
Value [2]interface{} `json:"value"`
|
||
} `json:"result"`
|
||
} `json:"data"`
|
||
}
|
||
if json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Data.Result) == 0 {
|
||
return 0, false
|
||
}
|
||
if len(payload.Data.Result[0].Value) < 2 {
|
||
return 0, false
|
||
}
|
||
raw, ok := payload.Data.Result[0].Value[1].(string)
|
||
if !ok {
|
||
return 0, false
|
||
}
|
||
return parseFloatSafe(raw), true
|
||
}
|
||
|
||
// ---------- 单位解析 ----------
|
||
|
||
// parseQuantityMilli 解析 CPU 用量/限额为毫核。
|
||
//
|
||
// 注意 metrics-server 返回的 CPU 常常是**纳核**（如 "194533645n" = 194.5 毫核），
|
||
// 早先只处理了 "m" 后缀，导致 "194533645" 被当成 194 亿核，
|
||
// 页面出现「CPU 已达 limit 的 4863341125%」这种荒谬结论。
|
||
func parseQuantityMilli(s string) int64 {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" || s == "0" {
|
||
return 0
|
||
}
|
||
switch {
|
||
case strings.HasSuffix(s, "n"): // 纳核
|
||
return int64(parseFloatSafe(strings.TrimSuffix(s, "n")) / 1e6)
|
||
case strings.HasSuffix(s, "u"): // 微核
|
||
return int64(parseFloatSafe(strings.TrimSuffix(s, "u")) / 1e3)
|
||
case strings.HasSuffix(s, "m"): // 毫核
|
||
return int64(parseFloatSafe(strings.TrimSuffix(s, "m")))
|
||
default: // 整核
|
||
return int64(parseFloatSafe(s) * 1000)
|
||
}
|
||
}
|
||
|
||
func parseQuantityBytes(s string) int64 {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" {
|
||
return 0
|
||
}
|
||
units := []struct {
|
||
suffix string
|
||
mul float64
|
||
}{
|
||
{"Gi", 1024 * 1024 * 1024}, {"Mi", 1024 * 1024}, {"Ki", 1024},
|
||
{"G", 1e9}, {"M", 1e6}, {"K", 1e3},
|
||
}
|
||
for _, u := range units {
|
||
if strings.HasSuffix(s, u.suffix) {
|
||
return int64(parseFloatSafe(strings.TrimSuffix(s, u.suffix)) * u.mul)
|
||
}
|
||
}
|
||
return int64(parseFloatSafe(s))
|
||
}
|
||
|
||
func parseFloatSafe(s string) float64 {
|
||
var f float64
|
||
if _, err := fmt.Sscanf(strings.TrimSpace(s), "%g", &f); err != nil {
|
||
return 0
|
||
}
|
||
return f
|
||
}
|
||
|
||
// parsePercent 解析 "0.15%" / "12.5%" → 数值百分比
|
||
func parsePercent(s string) float64 {
|
||
return parseFloatSafe(strings.TrimSuffix(strings.TrimSpace(s), "%"))
|
||
}
|
||
|
||
// parseDockerMemUsage 解析 docker stats 的 "120MiB / 7.6GiB" → (used, limit)
|
||
func parseDockerMemUsage(s string) (int64, int64) {
|
||
parts := strings.Split(s, "/")
|
||
if len(parts) != 2 {
|
||
return 0, 0
|
||
}
|
||
return parseDockerSize(parts[0]), parseDockerSize(parts[1])
|
||
}
|
||
|
||
func parseDockerSize(s string) int64 {
|
||
s = strings.TrimSpace(s)
|
||
units := []struct {
|
||
suffix string
|
||
mul float64
|
||
}{
|
||
{"GiB", 1024 * 1024 * 1024}, {"MiB", 1024 * 1024}, {"KiB", 1024},
|
||
{"GB", 1e9}, {"MB", 1e6}, {"KB", 1e3}, {"B", 1},
|
||
}
|
||
for _, u := range units {
|
||
if strings.HasSuffix(s, u.suffix) {
|
||
return int64(parseFloatSafe(strings.TrimSuffix(s, u.suffix)) * u.mul)
|
||
}
|
||
}
|
||
return int64(parseFloatSafe(s))
|
||
}
|
||
|
||
func nextCPU(currentMilli int64) int64 {
|
||
for _, c := range []int64{1000, 2000, 4000, 6000, 8000, 16000} {
|
||
if c > currentMilli {
|
||
return c
|
||
}
|
||
}
|
||
return currentMilli * 2
|
||
}
|
||
|
||
func nextMem(currentBytes int64) int64 {
|
||
for _, g := range []int64{2, 4, 8, 16, 32, 64} {
|
||
if g*1024*1024*1024 > currentBytes {
|
||
return g * 1024 * 1024 * 1024
|
||
}
|
||
}
|
||
return currentBytes * 2
|
||
}
|
||
|
||
func formatMilli(milli int64) string {
|
||
if milli%1000 == 0 {
|
||
return fmt.Sprintf("%d", milli/1000)
|
||
}
|
||
return fmt.Sprintf("%.1f", float64(milli)/1000)
|
||
}
|
||
|
||
func formatBytesShort(b int64) string {
|
||
if b >= 1024*1024*1024 {
|
||
return fmt.Sprintf("%dGi", b/(1024*1024*1024))
|
||
}
|
||
return fmt.Sprintf("%dMi", b/(1024*1024))
|
||
}
|