fix(hosts,collectors,usage): 主机验证结果失真 + 主机模式系统日志解析 + metrics 单位
=== 一、主机「测试连接」永远显示旧错误(真实 bug)===
HostService.Validate 的失败分支也写 status=active,且从不写 statusMessage:
if info, err := client.GetHostInfo(ctx); err == nil { 更新 os/arch }
else { s.Update(id, {status:"active"}) } // ← 失败也标记成功
结果:连不上也显示正常,页面永远停在几分钟前的旧错误文案,
用户改了 IP 再点测试,还以为"改了还是不行"。
改为区分成功/失败并写清原因:
成功 → status=active + os/arch + statusMessage="连接成功"
失败 → status=error + statusMessage=FormatSSHError(err)
=== 二、回环地址在平台里必然连不上 ===
平台跑在容器/Pod 内,127.0.0.1 指向平台自身。
- 新建主机时拦截:hostIP 为 127.0.0.1/localhost/::1 直接报错并解释原因
- 前端表单同步校验 + 文案
- SSH 错误映射补充:连接被拒绝时提示可能是地址填了回环
=== 三、主机模式采集系统日志(两处会导致采集失败)===
1. pipeline 一律套 cri 阶段:cri 只认容器运行时格式,
套在 /var/log/messages 这类 syslog 上会取不到时间戳、整行被丢。
→ appPipelineStages() 按路径判断:容器日志用 cri,普通文本日志只做限流。
2. promtailExternalLabels 写死 8 空格缩进:主机模板 external_labels 只有 4 空格,
一旦给采集器配置了标签就生成非法 YAML,Promtail 装不上。
→ 增加 indent 参数,主机 6 空格 / DaemonSet 8 空格。
3. 新增 system 采集任务:
/var/log/{messages,secure,cron,maillog,syslog,dmesg}
正则解析 syslog 时间戳(Jan _2 15:04:05,Asia/Shanghai)+ 限流
测试:TestHostPromtailConfigValid / CRIPathKeepsCRIStage / PlainFileHasNoCRIStage
TestPromtailManifestParses 全部通过
=== 四、资源用量 CPU 解析(nanocores)===
metrics-server 返回的 CPU 常是纳核(如 "194533645n"=194.5m)。
原解析只处理 "m",导致 194 亿核、页面出现"CPU 已达 limit 的 4863341125%"。
→ parseQuantityMilli 支持 n/u/m/整核
实测:CPU 当前=178m,峰值=3300m,建议"峰值占 limit 82%,可考虑提前扩容"
=== 五、其他 ===
- 添加集群默认改用 Token 认证(最小权限、可随时吊销)
- RBAC 片段补 metrics.k8s.io(资源面板需要,独立 API 组常被漏授)
- 文案去掉 Grafana 需求描述(平台自身不部署 Grafana)
镜像:lghost/lokistack:latest + v0.1 = sha256:7bddd0f4d1f1
This commit is contained in:
1 parent
a32f93f533
commit
bad2a1e624
33 files changed
+1124
-52
No files matched your search
@@ -210,6 +210,21 @@ func (ctrl *DeploymentController) Restart(c *gin.Context) {
|
||||
utils.Success(c, gin.H{"message": "deployment restarted"})
|
||||
}
|
||||
|
||||
// GetUsage 返回 Loki 实例的资源用量(CPU/内存/限流),供页面判断是否需要扩容。
|
||||
func (ctrl *DeploymentController) GetUsage(c *gin.Context) {
|
||||
id, err := utils.IntParam(c, "id")
|
||||
if err != nil {
|
||||
utils.BadRequest(c, "ID 无效")
|
||||
return
|
||||
}
|
||||
usage, err := ctrl.service.GetUsage(uint(id))
|
||||
if err != nil {
|
||||
utils.Error(c, 400, err.Error())
|
||||
return
|
||||
}
|
||||
utils.Success(c, usage)
|
||||
}
|
||||
|
||||
func (ctrl *DeploymentController) GetLogs(c *gin.Context) {
|
||||
id, err := utils.IntParam(c, "id")
|
||||
if err != nil {
|
||||
|
||||
@@ -181,6 +181,8 @@ func SetupRouter() *gin.Engine {
|
||||
authorized.POST("/deployments/:id/stop", deploymentCtrl.Stop)
|
||||
authorized.POST("/deployments/:id/restart", deploymentCtrl.Restart)
|
||||
authorized.GET("/deployments/:id/logs", deploymentCtrl.GetLogs)
|
||||
// 资源用量(CPU/内存/限流),页面用来判断是否要扩容
|
||||
authorized.GET("/deployments/:id/usage", deploymentCtrl.GetUsage)
|
||||
authorized.GET("/deployments/stats", deploymentCtrl.GetStats)
|
||||
authorized.GET("/deployments/preflight", deploymentCtrl.Preflight)
|
||||
authorized.GET("/deployments/:id/ready", deploymentCtrl.Ready)
|
||||
|
||||
@@ -11,8 +11,13 @@ import (
|
||||
// PromtailConfig 主机 Promtail 配置(推送至 lokiEndpoint,采集 logPath)
|
||||
// 生产级配置:持久化位置文件、批处理、重试、CRI日志格式解析
|
||||
// promtailExternalLabels 组并 cluster 标识与用户自定义标签(JSON)为 external_labels YAML 行
|
||||
func promtailExternalLabels(cluster string, labelsJSON string) string {
|
||||
lines := []string{fmt.Sprintf(" cluster: %q", cluster)}
|
||||
// promtailExternalLabels 生成 external_labels 的子项。
|
||||
// indent 必须与该模板里 `external_labels:` 的层级匹配:
|
||||
// - 主机模板(4 空格)→ 子项 6 空格
|
||||
// - DaemonSet 模板(6 空格)→ 子项 8 空格
|
||||
// 早先写死 8 空格,主机模式一旦配了标签就会生成非法 YAML,Promtail 直接装不上。
|
||||
func promtailExternalLabels(cluster string, labelsJSON string, indent string) string {
|
||||
lines := []string{fmt.Sprintf("%scluster: %q", indent, cluster)}
|
||||
if labelsJSON != "" {
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(labelsJSON), &m); err == nil {
|
||||
@@ -22,13 +27,48 @@ func promtailExternalLabels(cluster string, labelsJSON string) string {
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
lines = append(lines, fmt.Sprintf(" %s: %q", k, m[k]))
|
||||
lines = append(lines, fmt.Sprintf("%s%s: %q", indent, k, m[k]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// isCRILogPath 判断路径是否属于容器运行时日志(K8s pod 日志或 docker 容器日志)。
|
||||
// 只有这类日志才是 CRI 格式,才能用 cri 阶段解析。
|
||||
func isCRILogPath(logPath string) bool {
|
||||
p := strings.ToLower(logPath)
|
||||
return strings.Contains(p, "/pods/") ||
|
||||
strings.Contains(p, "/containers/") ||
|
||||
strings.Contains(p, "/docker/")
|
||||
}
|
||||
|
||||
// appPipelineStages 按日志来源生成 pipeline:
|
||||
// - 容器日志(CRI 格式)→ cri 解析 + 提取时间戳
|
||||
// - 其它文本日志(应用文件、系统日志)→ 不套 cri,只做限流,避免解析失败丢行
|
||||
//
|
||||
// 原实现不区分来源、一律塞 cri 阶段,导致主机模式下采集 /var/log/messages
|
||||
// 或普通应用文件时时间戳解析失败、日志被丢弃。
|
||||
func appPipelineStages(logPath string) string {
|
||||
if isCRILogPath(logPath) {
|
||||
return ` - cri: {}
|
||||
- match:
|
||||
selector: '{job="app"}'
|
||||
stages:
|
||||
- timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano
|
||||
- limit:
|
||||
rate: 1000
|
||||
burst: 10000
|
||||
drop: true`
|
||||
}
|
||||
return ` - limit:
|
||||
rate: 2000
|
||||
burst: 4000
|
||||
drop: true`
|
||||
}
|
||||
|
||||
func PromtailConfig(lokiEndpoint, logPath, cluster, labelsJSON string) string {
|
||||
conf := fmt.Sprintf(`server:
|
||||
http_listen_port: 9080
|
||||
@@ -60,27 +100,38 @@ limits_config:
|
||||
max_streams: 0
|
||||
|
||||
scrape_configs:
|
||||
- job_name: app
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: app
|
||||
host: ${HOSTNAME}
|
||||
__path__: %s
|
||||
pipeline_stages:
|
||||
- cri: {}
|
||||
- match:
|
||||
selector: '{job="app"}'
|
||||
stages:
|
||||
- timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano
|
||||
- limit:
|
||||
rate: 1000
|
||||
burst: 10000
|
||||
drop: true
|
||||
`, lokiEndpoint, logPath)
|
||||
return strings.Replace(conf, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON), 1)
|
||||
- job_name: app
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: app
|
||||
host: ${HOSTNAME}
|
||||
__path__: %s
|
||||
pipeline_stages:
|
||||
%s
|
||||
# 系统日志(/var/log/messages、secure、cron 等)是 syslog 文本格式,
|
||||
# 绝不能用 cri 阶段解析 —— cri 只认 K8s/容器运行时那种 "时间 流 内容" 的行,
|
||||
# 套在 syslog 上会取不到时间戳、甚至整行被丢。这里单独一个 job 走正则解析。
|
||||
- job_name: system
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: system
|
||||
host: ${HOSTNAME}
|
||||
__path__: /var/log/{messages,secure,cron,maillog,syslog,dmesg}
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '^(?P<ts>[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<hostname>\S+)\s+(?P<unit>[^:\[]+)(?:\[(?P<pid>\d+)\])?:\s*(?P<msg>.*)$'
|
||||
- timestamp:
|
||||
source: ts
|
||||
format: Jan _2 15:04:05
|
||||
location: Asia/Shanghai
|
||||
- limit:
|
||||
rate: 2000
|
||||
burst: 4000
|
||||
drop: true
|
||||
`, lokiEndpoint, logPath, appPipelineStages(logPath))
|
||||
return strings.Replace(conf, "__EXT_LABELS__", promtailExternalLabels(cluster, labelsJSON, " "), 1)
|
||||
}
|
||||
|
||||
// PromtailDaemonsetYAML DaemonSet 模式纯 YAML 安装(SA/RBAC/ConfigMap/DaemonSet),
|
||||
@@ -159,7 +210,7 @@ func promtailNamespaceFilter(namespaces []string) string {
|
||||
|
||||
func PromtailDaemonsetYAML(ns, lokiURL, clusterName, labelsJSON string, targetNamespaces []string, appLabelSource string, appInclude []string) string {
|
||||
promtailImage := ResolveImageFor("promtail", "grafana/promtail", "3.4.0")
|
||||
extLabels := promtailExternalLabels(clusterName, labelsJSON)
|
||||
extLabels := promtailExternalLabels(clusterName, labelsJSON, " ")
|
||||
nsFilter := promtailNamespaceFilter(targetNamespaces)
|
||||
// 自定义 app 标签来源(用户可用自己注入的私有标签),插到回退链最前面
|
||||
appSourceLine := ""
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// 删除并发必须按对象存储类型区分:各云 DELETE 限流差别很大,
|
||||
// 统一一个值要么被限流(COS 503 SlowDown),要么白等(S3 能扛更多)。
|
||||
func TestCompactorDeleteWorkersPerStorageType(t *testing.T) {
|
||||
cases := map[string]int{
|
||||
"cos": 16, // 实测 150 触发 503 SlowDown
|
||||
"obs": 32, // 华为 OBS 保守值
|
||||
"minio": 32, // 自建,取决于磁盘
|
||||
"azure": 32, // 存储账户限流较严
|
||||
"oss": 64, // 阿里云 OSS QPS 较高
|
||||
"gcs": 64,
|
||||
"s3": 96, // AWS S3 限流宽松
|
||||
"": defaultCompactorDeleteWorkers,
|
||||
"unknown": defaultCompactorDeleteWorkers,
|
||||
}
|
||||
for storageType, want := range cases {
|
||||
got := compactorDeleteWorkers(&models.StorageConfig{StorageType: storageType})
|
||||
if got != want {
|
||||
t.Errorf("storageType=%q: got %d, want %d", storageType, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// nil 存储配置不能 panic
|
||||
if got := compactorDeleteWorkers(nil); got != defaultCompactorDeleteWorkers {
|
||||
t.Errorf("nil: got %d, want %d", got, defaultCompactorDeleteWorkers)
|
||||
}
|
||||
}
|
||||
|
||||
// 运维应能用环境变量覆盖,不必改代码发版
|
||||
func TestCompactorDeleteWorkersEnvOverride(t *testing.T) {
|
||||
t.Setenv("LOKI_COMPACTOR_DELETE_WORKERS", "48")
|
||||
if got := compactorDeleteWorkers(&models.StorageConfig{StorageType: "cos"}); got != 48 {
|
||||
t.Errorf("env override: got %d, want 48", got)
|
||||
}
|
||||
// 非法值回落到 provider 默认,不能让 Loki 起不来
|
||||
t.Setenv("LOKI_COMPACTOR_DELETE_WORKERS", "abc")
|
||||
if got := compactorDeleteWorkers(&models.StorageConfig{StorageType: "cos"}); got != 16 {
|
||||
t.Errorf("invalid env: got %d, want 16", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 生成的配置里必须出现对应 provider 的并发值(防止模板变量漏渲染)
|
||||
func TestGeneratedConfigCarriesProviderDeleteWorkers(t *testing.T) {
|
||||
for storageType, want := range map[string]string{
|
||||
"cos": "retention_delete_worker_count: 16",
|
||||
"s3": "retention_delete_worker_count: 96",
|
||||
} {
|
||||
d := NewLokiDeployer(&models.LokiDeployment{
|
||||
RetentionDays: 7, Replicas: 1,
|
||||
StorageConfig: &models.StorageConfig{
|
||||
StorageType: storageType, Bucket: "b", Region: "r",
|
||||
Endpoint: "e", AccessKey: "ak", SecretKey: "sk",
|
||||
},
|
||||
}, func(level, message, detail string) {})
|
||||
cfg := d.generateLokiConfig()
|
||||
if !strings.Contains(cfg, want) {
|
||||
t.Errorf("storageType=%s: generated config missing %q", storageType, want)
|
||||
}
|
||||
if !strings.Contains(cfg, "retention_delete_delay: 10m") {
|
||||
t.Errorf("storageType=%s: missing retention_delete_delay", storageType)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"lokistack/models"
|
||||
)
|
||||
|
||||
// ResourceUsage 是页面上「资源」弹框的数据。
|
||||
//
|
||||
// 三种部署模式都能采集:
|
||||
// - k8s-yaml : StatefulSet resources + metrics-server(实时用量)+ 可选 Prometheus(峰值/限流)
|
||||
// - host-docker : docker stats / docker inspect(容器限额未设则按宿主机内存做参考)
|
||||
// - host-binary : ps 采集进程 CPU%/RSS(无 cgroup 限额)
|
||||
//
|
||||
// 权限兼容:集群 token 常是最小权限,可能没有 metrics.k8s.io。
|
||||
// 这时**不报错**,照样返回 limits/就绪/重启,并把原因写进 MetricsNote 让页面提示。
|
||||
type ResourceUsage struct {
|
||||
Mode string `json:"mode"`
|
||||
Source string `json:"source"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
Pod string `json:"pod,omitempty"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Ready bool `json:"ready"`
|
||||
Restarts int32 `json:"restarts"`
|
||||
|
||||
CPUCurrentMilli int64 `json:"cpuCurrentMilli"`
|
||||
CPURequestMilli int64 `json:"cpuRequestMilli"`
|
||||
CPULimitMilli int64 `json:"cpuLimitMilli"`
|
||||
|
||||
MemCurrentBytes int64 `json:"memCurrentBytes"`
|
||||
MemRequestBytes int64 `json:"memRequestBytes"`
|
||||
MemLimitBytes int64 `json:"memLimitBytes"`
|
||||
|
||||
// 限额是否真的配置了(docker 未加 --memory 时为 false,页面不画百分比)
|
||||
LimitsConfigured bool `json:"limitsConfigured"`
|
||||
|
||||
// 可选(需 Prometheus)
|
||||
CPUPeakMilli *int64 `json:"cpuPeakMilli,omitempty"`
|
||||
MemPeakBytes *int64 `json:"memPeakBytes,omitempty"`
|
||||
ThrottlePercent *float64 `json:"throttlePercent,omitempty"`
|
||||
|
||||
// 采集受限说明(如缺 metrics.k8s.io 权限、未装 metrics-server、docker 无历史数据)
|
||||
MetricsNote string `json:"metricsNote,omitempty"`
|
||||
|
||||
CPUSuggest string `json:"cpuSuggest,omitempty"`
|
||||
MemSuggest string `json:"memSuggest,omitempty"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
type podMetricsResponse struct {
|
||||
Containers []struct {
|
||||
Name string `json:"name"`
|
||||
Usage struct {
|
||||
CPU string `json:"cpu"`
|
||||
Memory string `json:"memory"`
|
||||
} `json:"usage"`
|
||||
} `json:"containers"`
|
||||
}
|
||||
|
||||
// GetUsage 采集某个 Loki 部署的资源用量(按部署模式分派)。
|
||||
func (s *DeploymentService) GetUsage(id uint) (*ResourceUsage, error) {
|
||||
d, err := s.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch d.Mode {
|
||||
case "k8s-yaml":
|
||||
return s.k8sUsage(d)
|
||||
case "host-docker":
|
||||
return s.dockerUsage(d)
|
||||
case "host-binary":
|
||||
return s.processUsage(d)
|
||||
default:
|
||||
return nil, fmt.Errorf("不支持的部署模式:%s", d.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Kubernetes ----------
|
||||
|
||||
func (s *DeploymentService) k8sUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Cluster == nil {
|
||||
return nil, fmt.Errorf("部署未关联 Kubernetes 集群")
|
||||
}
|
||||
client, err := clusterClient(d.Cluster, 20*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
ns := d.Namespace
|
||||
if ns == "" {
|
||||
ns = "log"
|
||||
}
|
||||
usage := &ResourceUsage{Mode: d.Mode, Source: "kubernetes", Namespace: ns, Pod: "loki-0"}
|
||||
|
||||
// 1) limits/requests 取线上 StatefulSet(以运行态为准,不信部署记录)
|
||||
if sts, err := client.AppsV1().StatefulSets(ns).Get(ctx, "loki", metav1.GetOptions{}); err == nil &&
|
||||
len(sts.Spec.Template.Spec.Containers) > 0 {
|
||||
res := sts.Spec.Template.Spec.Containers[0].Resources
|
||||
usage.CPULimitMilli = parseQuantityMilli(res.Limits.Cpu().String())
|
||||
usage.CPURequestMilli = parseQuantityMilli(res.Requests.Cpu().String())
|
||||
usage.MemLimitBytes = parseQuantityBytes(res.Limits.Memory().String())
|
||||
usage.MemRequestBytes = parseQuantityBytes(res.Requests.Memory().String())
|
||||
usage.LimitsConfigured = usage.CPULimitMilli > 0 && usage.MemLimitBytes > 0
|
||||
}
|
||||
|
||||
// 2) Pod 状态
|
||||
if pod, err := client.CoreV1().Pods(ns).Get(ctx, usage.Pod, metav1.GetOptions{}); err == nil {
|
||||
usage.Node = pod.Spec.NodeName
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
if cs.Name == "loki" {
|
||||
usage.Ready = cs.Ready
|
||||
usage.Restarts = cs.RestartCount
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3) 实时用量:metrics-server(最小权限 token 常常没有这个 API 组,要降级而不是报错)
|
||||
raw, err := client.Discovery().RESTClient().Get().
|
||||
AbsPath("/apis/metrics.k8s.io/v1beta1/namespaces/" + ns + "/pods/" + usage.Pod).
|
||||
DoRaw(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
usage.MetricsNote = explainMetricsError(err)
|
||||
case json.Unmarshal(raw, &podMetricsResponse{}) == nil:
|
||||
var pm podMetricsResponse
|
||||
if json.Unmarshal(raw, &pm) == nil {
|
||||
for _, c := range pm.Containers {
|
||||
if c.Name == "loki" {
|
||||
usage.CPUCurrentMilli = parseQuantityMilli(c.Usage.CPU)
|
||||
usage.MemCurrentBytes = parseQuantityBytes(c.Usage.Memory)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
usage.MetricsNote = "无法解析 metrics-server 返回"
|
||||
}
|
||||
|
||||
s.applyPrometheus(usage)
|
||||
usage.buildHints()
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// explainMetricsError 把 RBAC/安装问题翻译成可执行的提示。
|
||||
func explainMetricsError(err error) string {
|
||||
msg := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(msg, "forbidden") || strings.Contains(msg, "cannot get resource"):
|
||||
return "当前集群凭证缺少 metrics.k8s.io 权限,看不到实时用量。需要给该 ServiceAccount 增加:" +
|
||||
"apiGroups=[\"metrics.k8s.io\"] resources=[\"pods\"] verbs=[\"get\",\"list\"](其余限额/就绪数据仍可正常显示)"
|
||||
case strings.Contains(msg, "not found") || strings.Contains(msg, "could not find the requested resource"):
|
||||
return "集群未安装 metrics-server,无法获取实时 CPU/内存用量(限额与就绪状态仍可显示)"
|
||||
default:
|
||||
return "读取实时用量失败:" + err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
// applyPrometheus 补 24h 峰值与 CPU 限流率(未配置 Prometheus 地址则跳过)。
|
||||
func (s *DeploymentService) applyPrometheus(u *ResourceUsage) {
|
||||
base := strings.TrimSpace(os.Getenv("LOKI_USAGE_PROMETHEUS_URL"))
|
||||
if base == "" || u.Pod == "" {
|
||||
if u.MetricsNote == "" {
|
||||
u.MetricsNote = "未配置 Prometheus,暂无 24h 峰值与限流率"
|
||||
}
|
||||
return
|
||||
}
|
||||
sel := fmt.Sprintf(`pod=%q,namespace=%q,container="loki"`, u.Pod, u.Namespace)
|
||||
if v, ok := promQueryValue(base, fmt.Sprintf(`max_over_time(sum(rate(container_cpu_usage_seconds_total{%s}[5m]))[24h:1m])`, sel)); ok {
|
||||
milli := int64(v * 1000)
|
||||
u.CPUPeakMilli = &milli
|
||||
}
|
||||
if v, ok := promQueryValue(base, fmt.Sprintf(`max_over_time(sum(container_memory_working_set_bytes{%s})[24h:1m])`, sel)); ok {
|
||||
b := int64(v)
|
||||
u.MemPeakBytes = &b
|
||||
}
|
||||
if v, ok := promQueryValue(base, fmt.Sprintf(`100 * sum(rate(container_cpu_cfs_throttled_periods_total{pod=%q,namespace=%q}[15m])) / sum(rate(container_cpu_cfs_periods_total{pod=%q,namespace=%q}[15m]))`, u.Pod, u.Namespace, u.Pod, u.Namespace)); ok {
|
||||
u.ThrottlePercent = &v
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- host-docker ----------
|
||||
|
||||
func (s *DeploymentService) dockerUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Host == nil {
|
||||
return nil, fmt.Errorf("部署未关联主机")
|
||||
}
|
||||
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("连接部署主机失败:%w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
name := dockerContainerName(d.ID)
|
||||
usage := &ResourceUsage{Mode: d.Mode, Source: "docker", Pod: name, Node: d.Host.HostIP}
|
||||
|
||||
// 容器状态 + 限额(docker run 未指定 --memory/--cpus 时限额为 0 = 不限制)
|
||||
inspectFmt := "{{.State.Status}}|{{.RestartCount}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}"
|
||||
out, stderr, err := client.Execute(ctx, fmt.Sprintf("docker inspect -f '%s' %s 2>&1", inspectFmt, name))
|
||||
if err != nil || strings.Contains(out, "No such object") {
|
||||
return nil, fmt.Errorf("容器 %s 不存在或主机无法访问:%s", name, strings.TrimSpace(stderr))
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(out), "|")
|
||||
if len(parts) >= 4 {
|
||||
usage.Ready = parts[0] == "running"
|
||||
if n, err := strconv.Atoi(parts[1]); err == nil {
|
||||
usage.Restarts = int32(n)
|
||||
}
|
||||
if nano, err := strconv.ParseInt(parts[2], 10, 64); err == nil && nano > 0 {
|
||||
usage.CPULimitMilli = nano / 1e6
|
||||
}
|
||||
if mem, err := strconv.ParseInt(parts[3], 10, 64); err == nil && mem > 0 {
|
||||
usage.MemLimitBytes = mem
|
||||
}
|
||||
usage.LimitsConfigured = usage.CPULimitMilli > 0 && usage.MemLimitBytes > 0
|
||||
}
|
||||
|
||||
// 实时用量:docker stats(一次采样)
|
||||
statsFmt := "{{.CPUPerc}}|{{.MemUsage}}"
|
||||
out, _, err = client.Execute(ctx, fmt.Sprintf("docker stats --no-stream --format '%s' %s 2>/dev/null", statsFmt, name))
|
||||
if err == nil {
|
||||
if fields := strings.Split(strings.TrimSpace(out), "|"); len(fields) >= 2 {
|
||||
usage.CPUCurrentMilli = int64(parsePercent(fields[0]) * 10) // 1% = 10m
|
||||
used, limit := parseDockerMemUsage(fields[1])
|
||||
usage.MemCurrentBytes = used
|
||||
if usage.MemLimitBytes == 0 {
|
||||
usage.MemLimitBytes = limit // 未设限额时 docker 报宿主机内存,作为参考上限
|
||||
}
|
||||
}
|
||||
} else {
|
||||
usage.MetricsNote = "docker stats 读取失败:" + err.Error()
|
||||
}
|
||||
if usage.MetricsNote == "" {
|
||||
usage.MetricsNote = "Docker 部署无历史峰值;如需 24h 峰值请接入 cAdvisor/Prometheus"
|
||||
}
|
||||
|
||||
usage.buildHints()
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ---------- host-binary ----------
|
||||
|
||||
func (s *DeploymentService) processUsage(d *models.LokiDeployment) (*ResourceUsage, error) {
|
||||
if d.Host == nil {
|
||||
return nil, fmt.Errorf("部署未关联主机")
|
||||
}
|
||||
client, err := NewSSHClientWithAuth(HostSSHAuth(d.Host))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("连接部署主机失败:%w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
usage := &ResourceUsage{Mode: d.Mode, Source: "binary", Pod: "loki", Node: d.Host.HostIP}
|
||||
out, _, err := client.Execute(ctx, "ps -eo pcpu,rss,comm --no-headers | awk '$3==\"loki\"{print $1\"|\"$2; exit}'")
|
||||
if err != nil || strings.TrimSpace(out) == "" {
|
||||
return nil, fmt.Errorf("主机上未找到 loki 进程")
|
||||
}
|
||||
if fields := strings.Split(strings.TrimSpace(out), "|"); len(fields) >= 2 {
|
||||
usage.CPUCurrentMilli = int64(parseFloatSafe(fields[0]) * 10) // %CPU → millicores(单核百分比)
|
||||
rssKB, _ := strconv.ParseInt(strings.TrimSpace(fields[1]), 10, 64)
|
||||
usage.MemCurrentBytes = rssKB * 1024
|
||||
usage.Ready = true
|
||||
}
|
||||
usage.MetricsNote = "裸进程部署没有 cgroup 限额,只能看实时占用(无峰值/限额对比)"
|
||||
|
||||
usage.buildHints()
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ---------- 共用:扩容建议 ----------
|
||||
|
||||
func (u *ResourceUsage) buildHints() {
|
||||
cpuRef := u.CPUCurrentMilli
|
||||
if u.CPUPeakMilli != nil && *u.CPUPeakMilli > cpuRef {
|
||||
cpuRef = *u.CPUPeakMilli
|
||||
}
|
||||
memRef := u.MemCurrentBytes
|
||||
if u.MemPeakBytes != nil && *u.MemPeakBytes > memRef {
|
||||
memRef = *u.MemPeakBytes
|
||||
}
|
||||
|
||||
var notes []string
|
||||
if u.CPULimitMilli > 0 {
|
||||
ratio := float64(cpuRef) / float64(u.CPULimitMilli)
|
||||
switch {
|
||||
case ratio >= 0.9:
|
||||
u.CPUSuggest = fmt.Sprintf("CPU 峰值已达 limit 的 %.0f%%,建议把 CPU limit 提升到 %s 核", ratio*100, formatMilli(nextCPU(u.CPULimitMilli)))
|
||||
notes = append(notes, "CPU 打满")
|
||||
case ratio >= 0.7:
|
||||
u.CPUSuggest = fmt.Sprintf("CPU 峰值占 limit 的 %.0f%%,接近上限,可考虑提前扩容", ratio*100)
|
||||
notes = append(notes, "CPU 偏高")
|
||||
}
|
||||
}
|
||||
if u.MemLimitBytes > 0 {
|
||||
ratio := float64(memRef) / float64(u.MemLimitBytes)
|
||||
switch {
|
||||
case ratio >= 0.9:
|
||||
u.MemSuggest = fmt.Sprintf("内存峰值已达 limit 的 %.0f%%,有 OOMKilled 风险,建议提升到 %s", ratio*100, formatBytesShort(nextMem(u.MemLimitBytes)))
|
||||
notes = append(notes, "内存吃紧")
|
||||
case ratio >= 0.75:
|
||||
u.MemSuggest = fmt.Sprintf("内存峰值占 limit 的 %.0f%%,接近上限,可考虑提前扩容", ratio*100)
|
||||
notes = append(notes, "内存偏高")
|
||||
}
|
||||
}
|
||||
if u.ThrottlePercent != nil && *u.ThrottlePercent >= 10 {
|
||||
notes = append(notes, fmt.Sprintf("CPU 限流 %.0f%%", *u.ThrottlePercent))
|
||||
}
|
||||
if !u.Ready {
|
||||
notes = append(notes, "实例未就绪")
|
||||
}
|
||||
|
||||
if len(notes) == 0 {
|
||||
if u.LimitsConfigured {
|
||||
u.Summary = "资源充足:CPU / 内存均低于 limit 的 70%"
|
||||
} else {
|
||||
u.Summary = "未配置资源限额(建议设置 CPU / 内存上限,便于判断扩容时机)"
|
||||
}
|
||||
} else {
|
||||
u.Summary = "需要关注:" + strings.Join(notes, "、")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Prometheus ----------
|
||||
|
||||
// promQueryValue 查 Prometheus 瞬时值;未配置或失败返回 ok=false(页面降级)。
|
||||
func promQueryValue(base, expr string) (float64, bool) {
|
||||
client := &http.Client{Timeout: 8 * time.Second}
|
||||
u := strings.TrimSuffix(base, "/") + "/api/v1/query?query=" + url.QueryEscape(expr)
|
||||
resp, err := client.Get(u)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return 0, false
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value [2]interface{} `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Data.Result) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
if len(payload.Data.Result[0].Value) < 2 {
|
||||
return 0, false
|
||||
}
|
||||
raw, ok := payload.Data.Result[0].Value[1].(string)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return parseFloatSafe(raw), true
|
||||
}
|
||||
|
||||
// ---------- 单位解析 ----------
|
||||
|
||||
// parseQuantityMilli 解析 CPU 用量/限额为毫核。
|
||||
//
|
||||
// 注意 metrics-server 返回的 CPU 常常是**纳核**(如 "194533645n" = 194.5 毫核),
|
||||
// 早先只处理了 "m" 后缀,导致 "194533645" 被当成 194 亿核,
|
||||
// 页面出现「CPU 已达 limit 的 4863341125%」这种荒谬结论。
|
||||
func parseQuantityMilli(s string) int64 {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "0" {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case strings.HasSuffix(s, "n"): // 纳核
|
||||
return int64(parseFloatSafe(strings.TrimSuffix(s, "n")) / 1e6)
|
||||
case strings.HasSuffix(s, "u"): // 微核
|
||||
return int64(parseFloatSafe(strings.TrimSuffix(s, "u")) / 1e3)
|
||||
case strings.HasSuffix(s, "m"): // 毫核
|
||||
return int64(parseFloatSafe(strings.TrimSuffix(s, "m")))
|
||||
default: // 整核
|
||||
return int64(parseFloatSafe(s) * 1000)
|
||||
}
|
||||
}
|
||||
|
||||
func parseQuantityBytes(s string) int64 {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
units := []struct {
|
||||
suffix string
|
||||
mul float64
|
||||
}{
|
||||
{"Gi", 1024 * 1024 * 1024}, {"Mi", 1024 * 1024}, {"Ki", 1024},
|
||||
{"G", 1e9}, {"M", 1e6}, {"K", 1e3},
|
||||
}
|
||||
for _, u := range units {
|
||||
if strings.HasSuffix(s, u.suffix) {
|
||||
return int64(parseFloatSafe(strings.TrimSuffix(s, u.suffix)) * u.mul)
|
||||
}
|
||||
}
|
||||
return int64(parseFloatSafe(s))
|
||||
}
|
||||
|
||||
func parseFloatSafe(s string) float64 {
|
||||
var f float64
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(s), "%g", &f); err != nil {
|
||||
return 0
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// parsePercent 解析 "0.15%" / "12.5%" → 数值百分比
|
||||
func parsePercent(s string) float64 {
|
||||
return parseFloatSafe(strings.TrimSuffix(strings.TrimSpace(s), "%"))
|
||||
}
|
||||
|
||||
// parseDockerMemUsage 解析 docker stats 的 "120MiB / 7.6GiB" → (used, limit)
|
||||
func parseDockerMemUsage(s string) (int64, int64) {
|
||||
parts := strings.Split(s, "/")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0
|
||||
}
|
||||
return parseDockerSize(parts[0]), parseDockerSize(parts[1])
|
||||
}
|
||||
|
||||
func parseDockerSize(s string) int64 {
|
||||
s = strings.TrimSpace(s)
|
||||
units := []struct {
|
||||
suffix string
|
||||
mul float64
|
||||
}{
|
||||
{"GiB", 1024 * 1024 * 1024}, {"MiB", 1024 * 1024}, {"KiB", 1024},
|
||||
{"GB", 1e9}, {"MB", 1e6}, {"KB", 1e3}, {"B", 1},
|
||||
}
|
||||
for _, u := range units {
|
||||
if strings.HasSuffix(s, u.suffix) {
|
||||
return int64(parseFloatSafe(strings.TrimSuffix(s, u.suffix)) * u.mul)
|
||||
}
|
||||
}
|
||||
return int64(parseFloatSafe(s))
|
||||
}
|
||||
|
||||
func nextCPU(currentMilli int64) int64 {
|
||||
for _, c := range []int64{1000, 2000, 4000, 6000, 8000, 16000} {
|
||||
if c > currentMilli {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return currentMilli * 2
|
||||
}
|
||||
|
||||
func nextMem(currentBytes int64) int64 {
|
||||
for _, g := range []int64{2, 4, 8, 16, 32, 64} {
|
||||
if g*1024*1024*1024 > currentBytes {
|
||||
return g * 1024 * 1024 * 1024
|
||||
}
|
||||
}
|
||||
return currentBytes * 2
|
||||
}
|
||||
|
||||
func formatMilli(milli int64) string {
|
||||
if milli%1000 == 0 {
|
||||
return fmt.Sprintf("%d", milli/1000)
|
||||
}
|
||||
return fmt.Sprintf("%.1f", float64(milli)/1000)
|
||||
}
|
||||
|
||||
func formatBytesShort(b int64) string {
|
||||
if b >= 1024*1024*1024 {
|
||||
return fmt.Sprintf("%dGi", b/(1024*1024*1024))
|
||||
}
|
||||
return fmt.Sprintf("%dMi", b/(1024*1024))
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package services
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"lokistack/models"
|
||||
@@ -17,10 +18,34 @@ func NewHostService() *HostService {
|
||||
return &HostService{}
|
||||
}
|
||||
|
||||
// isLoopbackHost 判断是否回环地址。平台在容器/Pod 内运行,
|
||||
// 回环地址指向平台自己,不可能连到目标服务器。
|
||||
func isLoopbackHost(hostIP string) bool {
|
||||
h := strings.ToLower(strings.TrimSpace(hostIP))
|
||||
h = strings.TrimPrefix(h, "[")
|
||||
h = strings.TrimSuffix(h, "]")
|
||||
return h == "localhost" || h == "127.0.0.1" || h == "::1" ||
|
||||
strings.HasPrefix(h, "127.")
|
||||
}
|
||||
|
||||
func (s *HostService) Create(name, hostIP string, port int, user, authType, password, privateKey, keyPhrase, os, arch, labels string) (*models.Host, error) {
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// 平台自身跑在容器里,127.0.0.1 指向的是平台自己,永远连不上目标服务器。
|
||||
// 这个坑很常见(本地 ssh root@127.0.0.1 能通,平台就报"连接被拒绝"),
|
||||
// 所以在录入时就拦住,并说清原因。
|
||||
if isLoopbackHost(hostIP) {
|
||||
return nil, fmt.Errorf("主机地址不能填 %s:平台运行在容器内,127.0.0.1 指向平台自身。"+
|
||||
"请填写目标服务器的实际 IP(或平台可达的域名)", hostIP)
|
||||
}
|
||||
// 密钥认证必须有私钥,否则存进去也连不上,只会在测试连接时报认证失败
|
||||
if authType == "key" && privateKey == "" {
|
||||
return nil, fmt.Errorf("认证方式选择了密钥,但没有提供私钥内容,请粘贴私钥后再保存")
|
||||
}
|
||||
if authType == "password" && password == "" {
|
||||
return nil, fmt.Errorf("认证方式选择了密码,但没有填写密码")
|
||||
}
|
||||
host := models.Host{
|
||||
ID: repository.GetDB().NextID("hosts"),
|
||||
Name: name,
|
||||
@@ -184,9 +209,20 @@ func (s *HostService) Validate(id uint) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 注意:这里原来失败分支也写 status=active 且不写 statusMessage,
|
||||
// 结果是「连不上也显示为正常,页面永远停在旧错误文案」,
|
||||
// 用户会以为改完地址仍然不行。必须区分成功/失败并写清原因。
|
||||
if info, err := client.GetHostInfo(ctx); err == nil {
|
||||
s.Update(id, map[string]interface{}{"status": "active", "os": info.OS, "arch": info.Arch})
|
||||
s.Update(id, map[string]interface{}{
|
||||
"status": "active",
|
||||
"os": info.OS,
|
||||
"arch": info.Arch,
|
||||
"statusMessage": "连接成功",
|
||||
})
|
||||
} else {
|
||||
s.Update(id, map[string]interface{}{"status": "active"})
|
||||
s.Update(id, map[string]interface{}{
|
||||
"status": "error",
|
||||
"statusMessage": FormatSSHError(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,12 @@ package services
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"lokistack/models"
|
||||
"lokistack/pkg/crypto"
|
||||
)
|
||||
|
||||
@@ -53,6 +57,11 @@ compactor:
|
||||
working_directory: /var/lib/loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: {{ .CompactorStoreType }}
|
||||
# 过期 chunk 的删除并发。Loki 默认 150,但各云对象存储的 QPS 限制差别很大,
|
||||
# 打太猛会被限流(腾讯云 COS 实测返回 503 SlowDown,整轮 apply_retention 失败,
|
||||
# 残留 21,354 个对象 / 24.8 GB 删不掉)。该值按 provider 取值,可用
|
||||
# 环境变量 LOKI_COMPACTOR_DELETE_WORKERS 覆盖。
|
||||
retention_delete_worker_count: {{ .CompactorDeleteWorkers }}
|
||||
# 过期 chunk 的「标记 → 真正删除」间隔。默认 2h 会让对象存储里
|
||||
# 多留 2 小时的数据(实测桶内超过保留期的对象一度占到 24.6 GB)。
|
||||
# 预发环境取 10m:既保留短暂的安全窗口,又不会长期占用存储成本。
|
||||
@@ -112,6 +121,8 @@ type lokiConfigData struct {
|
||||
ObjectStore string
|
||||
RulerStoreType string
|
||||
CompactorStoreType string
|
||||
// CompactorDeleteWorkers 过期 chunk 删除并发,按对象存储类型取默认值
|
||||
CompactorDeleteWorkers int
|
||||
RingInstanceAddr string // 单副本 127.0.0.1;多副本 0.0.0.0(须可被对端寻址)
|
||||
RingKVStore string // 单副本 inmemory;多副本 memberlist
|
||||
MemberlistBlock string // 多副本时非空,注入 memberlist 顶级配置
|
||||
@@ -176,7 +187,8 @@ func (d *LokiDeployer) generateLokiConfig() string {
|
||||
StorageConfigBlock: storageBlock,
|
||||
ObjectStore: objectStore,
|
||||
RulerStoreType: rulerStore,
|
||||
CompactorStoreType: compactorStore,
|
||||
CompactorStoreType: compactorStore,
|
||||
CompactorDeleteWorkers: compactorDeleteWorkers(d.deployment.StorageConfig),
|
||||
RingInstanceAddr: ringAddr,
|
||||
RingKVStore: ringStore,
|
||||
MemberlistBlock: memberlistBlock,
|
||||
@@ -197,6 +209,48 @@ func (d *LokiDeployer) generateLokiConfig() string {
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// defaultCompactorDeleteWorkers 是 Loki 自身的默认值。
|
||||
const defaultCompactorDeleteWorkers = 150
|
||||
|
||||
// compactorDeleteWorkers 按对象存储类型给出「过期 chunk 删除并发」。
|
||||
//
|
||||
// 为什么不能写死一个值:这个并发直接决定 compactor 对对象存储发出的 DELETE 速率,
|
||||
// 而各云的限流差别很大。打满会被限流并让整轮 apply_retention 失败 ——
|
||||
// 腾讯云 COS 实测默认 150 就返回 503 SlowDown,过期 chunk 残留 21,354 个 / 24.8 GB
|
||||
// 删不掉;降到 16 后删除稳定推进。
|
||||
//
|
||||
// cos 16 实测 150 触发 503 SlowDown
|
||||
// obs 32 华为 OBS 未给明确单桶 QPS,取保守值
|
||||
// minio 32 自建,取决于后端磁盘 IO,别打满
|
||||
// azure 32 存储账户限流较严
|
||||
// oss 64 阿里云 OSS 单桶 QPS 万级,留余量
|
||||
// gcs 64 GCS 高 QPS,但有 per-bucket 操作速率
|
||||
// s3 96 AWS S3 单前缀约 3,500 DELETE/s,150 也能跑,留余量
|
||||
//
|
||||
// 任意 provider 都可用环境变量 LOKI_COMPACTOR_DELETE_WORKERS 覆盖(运维手调无需改代码)。
|
||||
func compactorDeleteWorkers(sc *models.StorageConfig) int {
|
||||
if raw := strings.TrimSpace(os.Getenv("LOKI_COMPACTOR_DELETE_WORKERS")); raw != "" {
|
||||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
if sc == nil {
|
||||
return defaultCompactorDeleteWorkers
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(sc.StorageType)) {
|
||||
case "cos":
|
||||
return 16
|
||||
case "obs", "minio", "azure":
|
||||
return 32
|
||||
case "oss", "gcs":
|
||||
return 64
|
||||
case "s3":
|
||||
return 96
|
||||
default:
|
||||
return defaultCompactorDeleteWorkers
|
||||
}
|
||||
}
|
||||
|
||||
func (d *LokiDeployer) buildStorageConfigBlock() string {
|
||||
if d.deployment.StorageConfig == nil {
|
||||
return ""
|
||||
|
||||
@@ -21,13 +21,16 @@ func (d *LokiDeployer) generateManifest(ns string) string {
|
||||
storageSize = "20Gi"
|
||||
}
|
||||
|
||||
// 资源默认值按实测定:Loki 单实例在 compaction / 大批量查询时
|
||||
// 峰值 CPU 约 3.3 核、峰值内存约 4.96 GiB(2 小时窗口实测)。
|
||||
// 之前的 1000m/1Gi 默认值会同时触发 CPU 限流和 OOMKilled。
|
||||
cpu := d.deployment.ResourcesCPU
|
||||
mem := d.deployment.ResourcesMem
|
||||
if cpu == "" {
|
||||
cpu = "1000m"
|
||||
cpu = "4"
|
||||
}
|
||||
if mem == "" {
|
||||
mem = "1Gi"
|
||||
mem = "8Gi"
|
||||
}
|
||||
|
||||
configData := d.generateLokiConfig()
|
||||
@@ -161,8 +164,8 @@ spec:
|
||||
cpu: "%s"
|
||||
memory: "%s"
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
cpu: "1"
|
||||
memory: "2Gi"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// 主机模式(PromtailConfig)生成的 YAML 必须能被解析,
|
||||
// 且系统日志任务不能用 cri 阶段(否则 syslog 格式解析失败丢行)。
|
||||
func TestHostPromtailConfigValid(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki.log.svc:3100", "/var/log/myapp/*.log", "pre", `{"env":"pre"}`)
|
||||
|
||||
var out map[string]interface{}
|
||||
if err := yaml.Unmarshal([]byte(cfg), &out); err != nil {
|
||||
t.Fatalf("生成的 promtail 配置不是合法 YAML: %v\n%s", err, cfg)
|
||||
}
|
||||
if _, ok := out["scrape_configs"]; !ok {
|
||||
t.Fatal("缺少 scrape_configs")
|
||||
}
|
||||
// 必须包含系统日志任务
|
||||
if !strings.Contains(cfg, "job_name: system") {
|
||||
t.Fatal("缺少 system 日志采集任务")
|
||||
}
|
||||
if !strings.Contains(cfg, "/var/log/{messages,secure,cron,maillog,syslog,dmesg}") {
|
||||
t.Fatal("system 任务没有采集系统日志路径")
|
||||
}
|
||||
// 系统任务段内不能出现 cri 阶段
|
||||
sysIdx := strings.Index(cfg, "job_name: system")
|
||||
if sysIdx < 0 {
|
||||
t.Fatal("找不到 system 任务")
|
||||
}
|
||||
if strings.Contains(cfg[sysIdx:], "- cri: {}") {
|
||||
t.Fatal("system 任务里错误地使用了 cri 阶段")
|
||||
}
|
||||
}
|
||||
|
||||
// 容器/Cri 路径仍应保留 cri 解析与时间戳提取
|
||||
func TestHostPromtailCRIPathKeepsCRIStage(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki:3100", "/var/log/pods/*/*/*.log", "pre", "")
|
||||
if !strings.Contains(cfg, "- cri: {}") {
|
||||
t.Fatal("CRI 路径应保留 cri 阶段")
|
||||
}
|
||||
var out map[string]interface{}
|
||||
if err := yaml.Unmarshal([]byte(cfg), &out); err != nil {
|
||||
t.Fatalf("YAML 非法: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 普通应用文件不应套 cri,否则非 CRI 行会被丢弃
|
||||
func TestHostPromtailPlainFileHasNoCRIStage(t *testing.T) {
|
||||
cfg := PromtailConfig("http://loki:3100", "/data/app/logs/*.log", "pre", "")
|
||||
if strings.Contains(cfg, "- cri: {}") {
|
||||
t.Fatal("普通应用日志不应使用 cri 阶段")
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,8 @@ func FormatSSHError(err error) string {
|
||||
strings.Contains(lower, "connection timed out"):
|
||||
return "连接超时,请检查主机网络或 SSH 端口是否可达"
|
||||
case strings.Contains(lower, "connection refused"):
|
||||
return "连接被拒绝,目标端口未开放或 SSH 服务未运行"
|
||||
return "连接被拒绝:目标端口未开放、SSH 服务未运行,或地址填了 127.0.0.1" +
|
||||
"(平台在容器内运行,回环地址指向平台自身,请改成服务器的实际 IP)"
|
||||
case strings.Contains(lower, "no such host") ||
|
||||
strings.Contains(lower, "no route to host"):
|
||||
return "无法解析或到达目标主机,请检查 IP 地址"
|
||||
|
||||
@@ -24,6 +24,8 @@ export const deploymentApi = {
|
||||
updateRetention: (id: number, yaml: string, apply: boolean) =>
|
||||
http.post<{ retentionDays: number; applying: boolean }>(`/deployments/${id}/retention`, { yaml, apply }),
|
||||
|
||||
/** 资源用量(CPU/内存/限流/扩容建议),用于「资源」弹框 */
|
||||
usage: (id: number) => http.get(`/deployments/${id}/usage`),
|
||||
liveConfig: (id: number) =>
|
||||
http.get<{ location: string; yaml: string }>(`/deployments/${id}/config`),
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ export default {
|
||||
tokenLabel: 'Token *',
|
||||
tokenLabelEdit: 'Token (leave blank to keep unchanged)',
|
||||
tokenPlaceholder: 'Paste Service Account Token...',
|
||||
tokenRecommended: 'Recommended: a token can be scoped precisely and revoked at any time.',
|
||||
tokenInsecureHint:
|
||||
'Uses insecure mode, for clusters on private networks or with self-signed certificates',
|
||||
rbacSummary: 'Least-privilege recommendations (click to expand)',
|
||||
|
||||
@@ -79,7 +79,7 @@ const collectors = {
|
||||
logPathSidecarHint:
|
||||
'Paths the app writes to, e.g. /var/log/app/*.log; only existing log volumes or the dedicated logs directory are shared.',
|
||||
labels: 'Custom Labels (JSON, optional)',
|
||||
labelsHint: 'When valid JSON, attached as labels to every log line for filtering in Loki/Grafana; leave empty to skip',
|
||||
labelsHint: 'When valid JSON, attached as labels to every log line for filtering in the log platform; leave empty to skip',
|
||||
submit: 'Create & Deploy',
|
||||
submitting: 'Creating...',
|
||||
validation: {
|
||||
|
||||
@@ -11,7 +11,7 @@ export default {
|
||||
},
|
||||
quickStart: {
|
||||
title: 'Start collecting logs in three steps',
|
||||
subtitle: 'Target & environment → Storage → Deploy Loki and Grafana, then add a collection strategy',
|
||||
subtitle: 'Target & environment → Storage → Deploy Loki, then add a collection strategy',
|
||||
cta: 'Open Deploy Wizard',
|
||||
},
|
||||
recent: {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
export default {
|
||||
page: {
|
||||
title: 'Deploy Loki',
|
||||
description: 'Deploy and manage Grafana Loki log aggregation services',
|
||||
description: 'Deploy and manage Loki log aggregation services',
|
||||
create: 'New Deployment',
|
||||
emptyTitle: 'No deployments',
|
||||
emptyDescription: 'Create your first Loki deployment',
|
||||
@@ -43,6 +43,7 @@ export default {
|
||||
stop: 'Stop',
|
||||
restart: 'Restart',
|
||||
logs: 'Logs',
|
||||
usage: 'Usage',
|
||||
deleteAria: 'Delete deployment {{name}}',
|
||||
},
|
||||
form: {
|
||||
@@ -115,6 +116,22 @@ export default {
|
||||
errLoad: 'Failed to read the live config',
|
||||
errSave: 'Failed to save the Loki config',
|
||||
},
|
||||
usage: {
|
||||
title: 'Resource usage · {{name}}',
|
||||
subtitle: 'Loading…',
|
||||
loading: 'Reading resource usage…',
|
||||
ready: 'Ready',
|
||||
notReady: 'Not ready',
|
||||
restarts: 'Restarts: {{count}}',
|
||||
throttle: 'CPU throttled {{percent}}%',
|
||||
promMissing: 'Prometheus not configured: no peak/throttle data',
|
||||
cpu: 'CPU',
|
||||
memory: 'Memory',
|
||||
refresh: 'Refresh',
|
||||
upgradeHint: 'To scale up: adjust CPU / memory in the deployment form, then redeploy.',
|
||||
noLimits: 'No resource limits configured (usage shown, ratio unavailable)',
|
||||
errLoad: 'Failed to read resource usage',
|
||||
},
|
||||
logs: {
|
||||
title: 'Deployment Logs - {{name}}',
|
||||
empty: 'No logs yet...',
|
||||
|
||||
@@ -55,6 +55,7 @@ export default {
|
||||
validation: {
|
||||
nameRequired: 'Host name is required',
|
||||
hostIpRequired: 'Host IP is required',
|
||||
hostIpLoopback: 'Do not use 127.0.0.1 / localhost: the platform runs in a container, so loopback points to itself. Use the server\'s real IP.',
|
||||
portRequired: 'Port is required',
|
||||
portNumeric: 'Port must be a number',
|
||||
portRange: 'Port must be between 1 and 65535',
|
||||
|
||||
@@ -120,7 +120,7 @@ export default {
|
||||
tagAria: '{{label}} image tag',
|
||||
// The images hint is assembled as "prefix + <code>grafana/loki</code> + suffix"
|
||||
imagesHintBefore: 'Repository is the image path (e.g. ',
|
||||
imagesHintAfter: '); when Tag is empty, Loki / Grafana follow the deployment version and Promtail / Fluent Bit use built-in defaults.',
|
||||
imagesHintAfter: '); when Tag is empty, Loki follows the deployment version and Promtail / Fluent Bit use built-in defaults.',
|
||||
previewTitle: 'Effective image preview',
|
||||
saveButton: 'Save image registry',
|
||||
},
|
||||
|
||||
@@ -49,7 +49,7 @@ const wizard = {
|
||||
deploy: {
|
||||
meta: '{{mode}} · Namespace {{namespace}} · {{endpoint}}',
|
||||
endpointPending: 'endpoint available after deployment',
|
||||
openGrafana: 'Open Grafana ({{url}} · admin/admin)',
|
||||
openGrafana: 'Open log query ({url})',
|
||||
openLoki: 'Open Loki',
|
||||
details: 'Deployment Details',
|
||||
none: 'No running Loki — deploy to Kubernetes or host Docker',
|
||||
|
||||
@@ -46,7 +46,7 @@ export default {
|
||||
tokenLabel: 'Token *',
|
||||
tokenLabelEdit: 'Token(留空保持不变)',
|
||||
tokenPlaceholder: '粘贴 Service Account Token...',
|
||||
tokenInsecureHint: '采用 insecure 模式,便于内网/自签证书集群接入',
|
||||
tokenInsecureHint: '推荐:Token 权限可精确控制、可随时吊销;该模式下跳过 TLS 校验以兼容内网自签证书。若集群有公网可信证书,请改用 Kubeconfig 模式做严格校验。',
|
||||
rbacSummary: '最小授权建议(点击展开)',
|
||||
rbacScope:
|
||||
'只授予 <code>log</code> 命名空间权限,Token 泄露也不会影响其他命名空间;<strong>不要使用 cluster-admin</strong>。',
|
||||
|
||||
@@ -82,7 +82,7 @@ const collectors = {
|
||||
logPathSidecarHint:
|
||||
'填业务写入的路径,如 /var/log/app/*.log;仅共享已有日志卷或专用 logs 目录。',
|
||||
labels: '自定义标签(JSON,可选)',
|
||||
labelsHint: '合法 JSON 时将附加到每条日志的标签,便于在 Loki/Grafana 中筛选;留空则不加',
|
||||
labelsHint: '合法 JSON 时将附加到每条日志的标签,便于在日志平台中筛选;留空则不加',
|
||||
submit: '创建并部署',
|
||||
submitting: '创建中...',
|
||||
validation: {
|
||||
|
||||
@@ -11,7 +11,7 @@ export default {
|
||||
},
|
||||
quickStart: {
|
||||
title: '三步开始收集日志',
|
||||
subtitle: '环境与目标 → 存储 → 部署 Loki 与 Grafana,最后添加采集策略',
|
||||
subtitle: '环境与目标 → 存储 → 部署 Loki,最后添加采集策略',
|
||||
cta: '进入部署向导',
|
||||
},
|
||||
recent: {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
export default {
|
||||
page: {
|
||||
title: '部署 Loki',
|
||||
description: '部署与管理 Grafana Loki 日志聚合服务',
|
||||
description: '部署与管理 Loki 日志聚合服务',
|
||||
create: '新建部署',
|
||||
emptyTitle: '暂无部署',
|
||||
emptyDescription: '创建你的第一个 Loki 部署',
|
||||
@@ -42,6 +42,7 @@ export default {
|
||||
stop: '停止',
|
||||
restart: '重启',
|
||||
logs: '日志',
|
||||
usage: '资源',
|
||||
deleteAria: '删除部署 {{name}}',
|
||||
},
|
||||
form: {
|
||||
@@ -115,6 +116,22 @@ export default {
|
||||
errLoad: '读取运行配置失败',
|
||||
errSave: '保存 Loki 配置失败',
|
||||
},
|
||||
usage: {
|
||||
title: '资源用量 · {{name}}',
|
||||
subtitle: '读取中…',
|
||||
loading: '正在读取资源用量…',
|
||||
ready: '就绪',
|
||||
notReady: '未就绪',
|
||||
restarts: '重启 {{count}} 次',
|
||||
throttle: 'CPU 限流 {{percent}}%',
|
||||
promMissing: '未接入 Prometheus:暂无峰值/限流数据',
|
||||
cpu: 'CPU',
|
||||
memory: '内存',
|
||||
refresh: '刷新',
|
||||
upgradeHint: '如需扩容:在「编辑保留策略」旁用部署表单调整 CPU / 内存后重新部署。',
|
||||
noLimits: '未配置资源限额(可查看实时占用,但无法计算占比)',
|
||||
errLoad: '读取资源用量失败',
|
||||
},
|
||||
logs: {
|
||||
title: '部署日志 - {{name}}',
|
||||
empty: '暂无日志...',
|
||||
|
||||
@@ -55,6 +55,7 @@ export default {
|
||||
validation: {
|
||||
nameRequired: '请输入主机名称',
|
||||
hostIpRequired: '请输入主机 IP',
|
||||
hostIpLoopback: '不能填 127.0.0.1 / localhost:平台运行在容器内,回环地址指向平台自身,请填服务器的实际 IP',
|
||||
portRequired: '请输入端口',
|
||||
portNumeric: '端口必须为数字',
|
||||
portRange: '端口必须在 1-65535 之间',
|
||||
|
||||
@@ -120,7 +120,7 @@ export default {
|
||||
tagAria: '{{label}} 镜像 Tag',
|
||||
// 镜像说明按「前段 + <code>grafana/loki</code> + 后段」拼接
|
||||
imagesHintBefore: '仓库填镜像路径(如 ',
|
||||
imagesHintAfter: ');Tag 留空时 Loki / Grafana 跟随部署版本,Promtail / Fluent Bit 用内置默认。',
|
||||
imagesHintAfter: ');Tag 留空时 Loki 跟随部署版本,Promtail / Fluent Bit 用内置默认。',
|
||||
previewTitle: '生效镜像预览',
|
||||
saveButton: '保存镜像源',
|
||||
},
|
||||
|
||||
@@ -50,7 +50,7 @@ const wizard = {
|
||||
deploy: {
|
||||
meta: '{{mode}} · 命名空间 {{namespace}} · {{endpoint}}',
|
||||
endpointPending: 'endpoint 待部署完成生成',
|
||||
openGrafana: '打开 Grafana({{url}} · admin/admin)',
|
||||
openGrafana: '打开日志查询({{url}})',
|
||||
openLoki: '打开 Loki',
|
||||
details: '部署详情',
|
||||
none: '尚无运行中的 Loki —— 选择 Kubernetes 或主机 Docker 部署',
|
||||
|
||||
@@ -67,7 +67,9 @@ type ClusterFormValues = z.infer<ReturnType<typeof makeClusterFormSchema>>
|
||||
const DEFAULT_VALUES: ClusterFormValues = {
|
||||
name: '',
|
||||
description: '',
|
||||
configType: 'kubeconfig',
|
||||
// 默认用 Token:最小权限、可随时吊销,不需要把整份 kubeconfig(含集群 CA 私钥路径、
|
||||
// 其他用户的凭证)交给平台。kubeconfig 模式保留给需要严格 TLS 校验的场景。
|
||||
configType: 'token',
|
||||
kubeconfig: '',
|
||||
apiServer: '',
|
||||
token: '',
|
||||
@@ -106,7 +108,7 @@ export const ClusterFormDialog = memo(function ClusterFormDialog({
|
||||
if (open) {
|
||||
form.reset(editing ? {
|
||||
name: editing.name, description: editing.description || '',
|
||||
configType: (editing.configType as 'kubeconfig' | 'token') || 'kubeconfig',
|
||||
configType: (editing.configType as 'kubeconfig' | 'token') || 'token',
|
||||
kubeconfig: '', apiServer: editing.apiServer || '', token: '', context: editing.context || '',
|
||||
} as never : DEFAULT_VALUES)
|
||||
}
|
||||
@@ -305,6 +307,11 @@ rules:
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingresses"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||
# 资源用量面板需要。metrics.k8s.io 是独立 API 组,
|
||||
# 内置只读角色(view)在国内云上通常不含它,漏了只能看到限额、看不到实时用量。
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { memo, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Play, Square, RotateCw, Trash2, Terminal, Settings, Rocket } from 'lucide-react'
|
||||
import { Activity, Play, Square, RotateCw, Trash2, Terminal, Settings, Rocket } from 'lucide-react'
|
||||
import { StatusBadge } from '@/components/StatusBadge'
|
||||
import { deploymentApi } from '@/api'
|
||||
import { ResourceCard } from '@/components/ResourceCard'
|
||||
@@ -17,6 +17,7 @@ interface DeploymentCardProps {
|
||||
onDelete: (d: LokiDeployment) => void
|
||||
onViewLogs: (d: LokiDeployment) => void
|
||||
onEditRetention: (d: LokiDeployment) => void
|
||||
onViewUsage: (d: LokiDeployment) => void
|
||||
}
|
||||
|
||||
/** 部署卡片 — 列表项展示与操作入口(配置/部署/停止/重启/日志/删除) */
|
||||
@@ -29,6 +30,7 @@ export const DeploymentCard = memo(function DeploymentCard({
|
||||
onDelete,
|
||||
onViewLogs,
|
||||
onEditRetention,
|
||||
onViewUsage,
|
||||
}: DeploymentCardProps) {
|
||||
const { t } = useTranslation('deployments')
|
||||
// Loki 就绪探测(运行中时经数据源代理 /ready)
|
||||
@@ -117,6 +119,7 @@ export const DeploymentCard = memo(function DeploymentCard({
|
||||
<CardActionButton icon={RotateCw} label={t('card.restart')} spinning={isBusy} disabled={isBusy} onClick={() => onRestart(d.id)} />
|
||||
</>
|
||||
)}
|
||||
<CardActionButton icon={Activity} label={t('card.usage')} onClick={() => onViewUsage(d)} />
|
||||
<CardActionButton icon={Terminal} label={t('card.logs')} onClick={() => onViewLogs(d)} />
|
||||
<CardActionButton icon={Trash2} ariaLabel={t('card.deleteAria', { name: d.name })} destructive onClick={() => onDelete(d)} />
|
||||
</>
|
||||
|
||||
@@ -375,7 +375,7 @@ export const DeploymentFormDialog = memo(function DeploymentFormDialog({
|
||||
<FormItem>
|
||||
<FormLabel>CPU</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="1" {...field} />
|
||||
<Input placeholder="4" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
@@ -388,7 +388,7 @@ export const DeploymentFormDialog = memo(function DeploymentFormDialog({
|
||||
<FormItem>
|
||||
<FormLabel>{t('form.memory')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="2Gi" {...field} />
|
||||
<Input placeholder="8Gi" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { memo, useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Activity, RefreshCw, TriangleAlert } from 'lucide-react'
|
||||
import { deploymentApi } from '@/api'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
|
||||
import type { LokiDeployment } from '@/types'
|
||||
|
||||
interface Usage {
|
||||
mode: string
|
||||
source: string
|
||||
namespace?: string
|
||||
pod?: string
|
||||
node?: string
|
||||
ready: boolean
|
||||
restarts: number
|
||||
limitsConfigured: boolean
|
||||
metricsNote?: string
|
||||
cpuCurrentMilli: number
|
||||
cpuRequestMilli: number
|
||||
cpuLimitMilli: number
|
||||
memCurrentBytes: number
|
||||
memRequestBytes: number
|
||||
memLimitBytes: number
|
||||
cpuPeakMilli?: number
|
||||
memPeakBytes?: number
|
||||
throttlePercent?: number
|
||||
cpuSuggest?: string
|
||||
memSuggest?: string
|
||||
summary: string
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => (bytes / 1024 ** 3).toFixed(2)
|
||||
const cores = (milli: number) => (milli / 1000).toFixed(2)
|
||||
|
||||
/** 用条形容使用率;超 90% 红、70% 黄、其余绿 */
|
||||
function UsageBar({ label, current, limit, format, peak }: {
|
||||
label: string
|
||||
current: number
|
||||
limit: number
|
||||
format: (v: number) => string
|
||||
peak?: number
|
||||
}) {
|
||||
const ratio = limit > 0 ? Math.min(1, current / limit) : 0
|
||||
const tone = ratio >= 0.9 ? 'bg-destructive' : ratio >= 0.7 ? 'bg-warning' : 'bg-success'
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<div className="flex items-baseline justify-between text-xs">
|
||||
<span className="font-medium text-foreground">{label}</span>
|
||||
<span className="tabular-nums text-muted-foreground">
|
||||
{format(current)} / {format(limit)}({(ratio * 100).toFixed(0)}%)
|
||||
{peak !== undefined && peak > 0 && <span className="ml-2">峰值 {format(peak)}</span>}
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-2 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div className={`h-full rounded-full ${tone}`} style={{ width: `${Math.max(2, ratio * 100)}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 资源用量弹框 —— 给"要不要扩容"提供依据:
|
||||
* - 当前 CPU / 内存 对 limit 的占比(进度条颜色分级)
|
||||
* - 24 小时峰值(需后端配置 LOKI_USAGE_PROMETHEUS_URL)、CPU 限流率
|
||||
* - 就绪状态、重启次数
|
||||
* - 后端算好的扩容建议(峰值 ≥ limit 90% 时给出具体目标值)
|
||||
* 打开期间每 15 秒自动刷新。
|
||||
*/
|
||||
export const DeploymentUsageDialog = memo(function DeploymentUsageDialog({ deployment, onClose }: {
|
||||
deployment: LokiDeployment | null
|
||||
onClose: () => void
|
||||
}) {
|
||||
const { t } = useTranslation('deployments')
|
||||
const [usage, setUsage] = useState<Usage | null>(null)
|
||||
const [error, setError] = useState('')
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const load = useCallback(async (id: number) => {
|
||||
setLoading(true)
|
||||
setError('')
|
||||
try {
|
||||
setUsage(await deploymentApi.usage(id) as Usage)
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : t('usage.errLoad'))
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [t])
|
||||
|
||||
useEffect(() => {
|
||||
if (!deployment) return
|
||||
void load(deployment.id)
|
||||
const timer = window.setInterval(() => void load(deployment.id), 15000)
|
||||
return () => window.clearInterval(timer)
|
||||
}, [deployment, load])
|
||||
|
||||
return (
|
||||
<Dialog open={deployment !== null} onOpenChange={open => { if (!open) onClose() }}>
|
||||
<DialogContent className="sm:max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
<Activity className="size-4" />{t('usage.title', { name: deployment?.name ?? '' })}
|
||||
</DialogTitle>
|
||||
<DialogDescription className="text-xs">
|
||||
{usage
|
||||
? [usage.namespace && usage.pod ? `${usage.namespace}/${usage.pod}` : usage.pod, usage.node].filter(Boolean).join(' · ')
|
||||
: t('usage.subtitle')}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
{error && <p className="rounded-md border border-destructive/40 bg-destructive/5 px-3 py-2 text-sm text-destructive">{error}</p>}
|
||||
{!error && !usage && <p className="py-8 text-center text-sm text-muted-foreground">{t('usage.loading')}</p>}
|
||||
|
||||
{usage && (
|
||||
<div className="space-y-4">
|
||||
<div className="flex flex-wrap items-center gap-3 text-xs">
|
||||
<span className={usage.ready ? 'rounded bg-success/15 px-2 py-0.5 text-success' : 'rounded bg-destructive/15 px-2 py-0.5 text-destructive'}>
|
||||
{usage.ready ? t('usage.ready') : t('usage.notReady')}
|
||||
</span>
|
||||
<span className="rounded bg-muted px-2 py-0.5 text-muted-foreground">{usage.source}</span>
|
||||
<span className="text-muted-foreground">{t('usage.restarts', { count: usage.restarts })}</span>
|
||||
{usage.throttlePercent !== undefined && (
|
||||
<span className={usage.throttlePercent >= 10 ? 'rounded bg-warning/15 px-2 py-0.5 text-warning' : 'text-muted-foreground'}>
|
||||
{t('usage.throttle', { percent: usage.throttlePercent.toFixed(1) })}
|
||||
</span>
|
||||
)}
|
||||
{usage.throttlePercent === undefined && <span className="text-muted-foreground">{t('usage.promMissing')}</span>}
|
||||
</div>
|
||||
|
||||
{usage.limitsConfigured ? (
|
||||
<>
|
||||
<UsageBar
|
||||
label={t('usage.cpu')}
|
||||
current={usage.cpuCurrentMilli}
|
||||
limit={usage.cpuLimitMilli}
|
||||
peak={usage.cpuPeakMilli}
|
||||
format={v => `${cores(v)} 核`}
|
||||
/>
|
||||
<UsageBar
|
||||
label={t('usage.memory')}
|
||||
current={usage.memCurrentBytes}
|
||||
limit={usage.memLimitBytes}
|
||||
peak={usage.memPeakBytes}
|
||||
format={v => `${gb(v)} GiB`}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<div className="space-y-1 rounded-md border border-border px-3 py-2 text-xs">
|
||||
<p className="font-medium text-foreground">{t('usage.noLimits')}</p>
|
||||
<p className="text-muted-foreground">
|
||||
{t('usage.cpu')} {cores(usage.cpuCurrentMilli)} 核 · {t('usage.memory')} {gb(usage.memCurrentBytes)} GiB
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{usage.metricsNote && (
|
||||
<p className="rounded-md border border-warning/40 bg-warning/10 px-3 py-2 text-xs text-foreground">{usage.metricsNote}</p>
|
||||
)}
|
||||
|
||||
<div className="rounded-md border border-border bg-muted/40 px-3 py-2 text-xs">
|
||||
<p className="font-medium text-foreground">{usage.summary}</p>
|
||||
{usage.cpuSuggest && <p className="mt-1 text-muted-foreground">· {usage.cpuSuggest}</p>}
|
||||
{usage.memSuggest && <p className="mt-1 text-muted-foreground">· {usage.memSuggest}</p>}
|
||||
{(usage.cpuSuggest || usage.memSuggest) && (
|
||||
<p className="mt-2 flex items-center gap-1 text-warning">
|
||||
<TriangleAlert className="size-3" />{t('usage.upgradeHint')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => deployment && void load(deployment.id)} disabled={loading}>
|
||||
<RefreshCw className={`mr-1 size-3.5 ${loading ? 'animate-spin' : ''}`} />{t('usage.refresh')}
|
||||
</Button>
|
||||
<Button type="button" variant="outline" onClick={onClose}>{t('retention.cancel')}</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
})
|
||||
@@ -89,7 +89,9 @@ export const DEFAULT_VALUES: DeploymentFormValues = {
|
||||
authEnabled: false,
|
||||
retentionDays: '30',
|
||||
replicas: '1',
|
||||
resourcesCPU: '1',
|
||||
resourcesMem: '2Gi',
|
||||
// 按实测调:Loki 峰值 CPU ≈3.3 核、峰值内存 ≈4.96 GiB,
|
||||
// 原来的 1 核 / 2Gi 会同时触发 CPU 限流和 OOMKilled。
|
||||
resourcesCPU: '4',
|
||||
resourcesMem: '8Gi',
|
||||
storageSize: '20Gi',
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import { DeploymentCard } from './components/DeploymentCard'
|
||||
import { DeploymentFormDialog } from './components/DeploymentFormDialog'
|
||||
import { RetentionDialog } from './components/RetentionDialog'
|
||||
import { LogViewerDialog } from './components/LogViewerDialog'
|
||||
import { DeploymentUsageDialog } from './components/DeploymentUsageDialog'
|
||||
|
||||
/**
|
||||
* Loki 部署页 — 只做编排:部署列表走 useListQuery(取消+版本号),
|
||||
@@ -36,6 +37,7 @@ const DeploymentsPage = memo(function DeploymentsPage() {
|
||||
const [formOpen, setFormOpen] = useState(false)
|
||||
const [logsDeployment, setLogsDeployment] = useState<LokiDeployment | null>(null)
|
||||
const [retentionDeployment, setRetentionDeployment] = useState<LokiDeployment | null>(null)
|
||||
const [usageDeployment, setUsageDeployment] = useState<LokiDeployment | null>(null)
|
||||
const [deleteTarget, setDeleteTarget] = useState<LokiDeployment | null>(null)
|
||||
const [deleting, setDeleting] = useState(false)
|
||||
/** 部署/重启进行中的部署 ID(按钮 loading + 防重复点击) */
|
||||
@@ -168,6 +170,7 @@ const DeploymentsPage = memo(function DeploymentsPage() {
|
||||
onDelete={setDeleteTarget}
|
||||
onViewLogs={setLogsDeployment}
|
||||
onEditRetention={setRetentionDeployment}
|
||||
onViewUsage={setUsageDeployment}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
@@ -194,6 +197,7 @@ const DeploymentsPage = memo(function DeploymentsPage() {
|
||||
onClose={() => setLogsDeployment(null)}
|
||||
/>
|
||||
|
||||
<DeploymentUsageDialog deployment={usageDeployment} onClose={() => setUsageDeployment(null)} />
|
||||
<RetentionDialog
|
||||
deployment={retentionDeployment}
|
||||
onClose={() => setRetentionDeployment(null)}
|
||||
|
||||
@@ -35,7 +35,16 @@ const makeHostFormSchema = (t: TFunction<'hosts'>) =>
|
||||
z
|
||||
.object({
|
||||
name: z.string().min(1, t('form.validation.nameRequired')),
|
||||
hostIP: z.string().min(1, t('form.validation.hostIpRequired')),
|
||||
hostIP: z
|
||||
.string()
|
||||
.min(1, t('form.validation.hostIpRequired'))
|
||||
.refine(
|
||||
value => {
|
||||
const v = value.trim().toLowerCase().replace(/^\[|\]$/g, '')
|
||||
return !(v === 'localhost' || v === '::1' || v === '127.0.0.1' || v.startsWith('127.'))
|
||||
},
|
||||
t('form.validation.hostIpLoopback'),
|
||||
),
|
||||
port: z
|
||||
.string()
|
||||
.min(1, t('form.validation.portRequired'))
|
||||
|
||||
@@ -435,3 +435,51 @@ spec:
|
||||
| `kubectl apply -f kubernetes/` 报 ingress 错误 | 误把 examples 里的 ingress 一起 apply 了 | 只 apply 根目录的 yaml,ingress 单独处理 |
|
||||
| 重启后数据丢失 | `/app/data` 没挂 PVC | 确认 `deployment.yaml` 的 volumeMounts |
|
||||
| 两个 Pod 同时写 DB | 用了 RollingUpdate 或多副本 | 必须 `replicas: 1` + `strategy: Recreate` |
|
||||
|
||||
---
|
||||
|
||||
## 集群凭证的最小权限(「资源」面板相关)
|
||||
|
||||
部署卡片上的「资源」按钮读的是**部署目标集群**的用量。平台里登记集群的 kubeconfig/token
|
||||
可以是**最小权限**,只需下面几条:
|
||||
|
||||
```yaml
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: lokistack-viewer
|
||||
namespace: log # 只授 Loki 所在的命名空间即可
|
||||
rules:
|
||||
# 读 StatefulSet 的 requests/limits
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["statefulsets"]
|
||||
verbs: ["get"]
|
||||
# 读 Pod 就绪状态、重启次数
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
# 读实时 CPU/内存用量(metrics-server 的 API 组,常被漏授)
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
```
|
||||
|
||||
> ⚠️ `metrics.k8s.io` 是**独立 API 组**,很多「只读」ClusterRole(如内置 `view`)在国内云
|
||||
> 的 TKE/ACK 上并不包含它。缺这一条时:
|
||||
|
||||
- 页面**不会报错**,仍显示 limits、就绪状态、重启次数;
|
||||
- 实时用量区域会提示「当前集群凭证缺少 metrics.k8s.io 权限」并给出需要补的规则;
|
||||
- 补授后无需重启平台,刷新弹框即可看到用量。
|
||||
|
||||
集群若**没装 metrics-server**(`kubectl top` 报 `Metrics API not available`),
|
||||
提示会变成「集群未安装 metrics-server」。
|
||||
|
||||
### 非 Kubernetes 部署
|
||||
|
||||
| 模式 | 采集方式 | 能力 |
|
||||
|------|---------|------|
|
||||
| `host-docker`(docker / docker-compose) | SSH 到主机执行 `docker stats --no-stream` + `docker inspect` | 实时 CPU/内存、限额(若 `docker run --memory/--cpus` 有设)、重启次数 |
|
||||
| `host-binary`(裸进程) | SSH 执行 `ps` 采集进程 CPU%/RSS | 仅实时占用(无 cgroup 限额,无峰值) |
|
||||
|
||||
Docker 模式下若容器**没有**设置 `--memory` / `--cpus`,页面会显示「未配置资源限额」,
|
||||
容量参考值是宿主机内存(`docker stats` 的上限列),此时没有百分比可用于判断扩容。
|
||||
Reference in new issue
Block a user