feat(k8s): M2 Pod Metrics 详情页接入 — ListPods 注入 metrics

后端: - 新增 buildPodMetricsMap() 公共辅助函数，聚合多容器 CPU/Memory - ListPods / listPodsWithNS / ListPodsAllNS 三个 handler 全部注入 - PodSummary 增加 cpu_usage_m / mem_usage_mi 字段前端: - KubernetesWorkloads Pod 表格增加 CPU/内存列 - NamespaceResourcePage Pod 表格增加 CPU/内存列验证: - 17 个 Pod 全部返回真实 metrics (argocd/demo-jar) - curl /kubernetes-clusters/1/pods → cpu_usage_m='2m' mem_usage_mi='61Mi' 文档: - ROADMAP M1→done, M2→done, 验收标准全部勾选 - TDD_PLAN M2 通过标准补充 - PROJECT_EVALUATION P0 缺失项移除
2026-06-23 00:54:54 +08:00
1 parent 8c272a6fa6
commit 4f1ffd1275
@@ -0,0 +1,336 @@
+package toolinstaller
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"ezops/internal/models"
+
+	"gorm.io/gorm"
+)
+
+// ToolInstaller handles lazy installation of devops tools.
+// Each tool gets a sync.Once to prevent concurrent installations.
+type ToolInstaller struct {
+	db     *gorm.DB
+	onces  map[string]*sync.Once
+	mu     sync.Mutex
+}
+
+// New creates a ToolInstaller.
+func New(db *gorm.DB) *ToolInstaller {
+	return &ToolInstaller{
+		db:    db,
+		onces: make(map[string]*sync.Once),
+	}
+}
+
+// ensureOnce returns a sync.Once for the given tool name.
+func (ti *ToolInstaller) ensureOnce(name string) *sync.Once {
+	ti.mu.Lock()
+	defer ti.mu.Unlock()
+	if o, ok := ti.onces[name]; ok {
+		return o
+	}
+	o := &sync.Once{}
+	ti.onces[name] = o
+	return o
+}
+
+// EnsureAvailable checks if a tool is available and installs it if configured for lazy loading.
+// Returns nil if the tool is ready, or an error if installation failed.
+func (ti *ToolInstaller) EnsureAvailable(toolName string) error {
+	// Find tool in DB
+	var tool models.DeployTool
+	if err := ti.db.Where("name = ?", toolName).First(&tool).Error; err != nil {
+		// Tool not in DB — try to find it by path
+		if _, lookErr := exec.LookPath(toolName); lookErr == nil {
+			return nil // available even without DB config
+		}
+		return fmt.Errorf("tool %s not configured in DB", toolName)
+	}
+
+	if !tool.Enabled {
+		return fmt.Errorf("tool %s is disabled", toolName)
+	}
+
+	// Check if already installed
+	if tool.InstallMethod == models.ToolPreinstall || tool.InstallMethod == models.ToolLazy {
+		if inst, err := ti.checkInstalled(tool); err == nil {
+			// Update status if needed
+			if tool.Status != models.ToolInstalled {
+				ti.updateStatus(tool.ID, models.ToolInstalled, inst, "")
+			}
+			return nil
+		}
+	}
+
+	switch tool.InstallMethod {
+	case models.ToolPreinstall:
+		return fmt.Errorf("tool %s should be preinstalled but not found", toolName)
+	case models.ToolDisabled:
+		return fmt.Errorf("tool %s is disabled", toolName)
+	case models.ToolLazy:
+		// Use sync.Once to prevent concurrent installs
+		var installErr error
+		once := ti.ensureOnce(toolName)
+		once.Do(func() {
+			installErr = ti.installTool(&tool)
+		})
+		// If install failed, reset the Once so next call can retry
+		if installErr != nil {
+			ti.mu.Lock()
+			delete(ti.onces, toolName)
+			ti.mu.Unlock()
+		}
+		return installErr
+	default:
+		return fmt.Errorf("unknown install method: %s", tool.InstallMethod)
+	}
+}
+
+// checkInstalled runs the tool's check command and returns the version string.
+func (ti *ToolInstaller) checkInstalled(tool models.DeployTool) (string, error) {
+	// First try LookPath
+	if tool.BinaryName != "" {
+		if _, err := exec.LookPath(tool.BinaryName); err != nil {
+			return "", fmt.Errorf("binary %s not found", tool.BinaryName)
+		}
+	}
+
+	// Run check command if provided
+	if tool.CheckCommand != "" {
+		cmd := exec.Command("sh", "-c", tool.CheckCommand)
+		out, err := cmd.Output()
+		if err != nil {
+			return "", err
+		}
+		return strings.TrimSpace(strings.Split(string(out), "\n")[0]), nil
+	}
+
+	return "unknown", nil
+}
+
+// installTool performs the actual installation.
+func (ti *ToolInstaller) installTool(tool *models.DeployTool) error {
+	log.Printf("[ToolInstaller] Installing %s (method=%s)...", tool.Name, tool.InstallMethod)
+
+	// Mark as installing
+	ti.updateStatus(tool.ID, models.ToolInstalling, "", "")
+
+	var err error
+	if tool.InstallScript != "" {
+		err = ti.runInstallScript(tool)
+	} else if tool.DownloadURL != "" {
+		err = ti.downloadBinary(tool)
+	} else {
+		err = fmt.Errorf("no install script or download URL configured for %s", tool.Name)
+	}
+
+	if err != nil {
+		errMsg := err.Error()
+		log.Printf("[ToolInstaller] ERROR installing %s: %s", tool.Name, errMsg)
+		ti.updateStatus(tool.ID, models.ToolFailed, "", errMsg)
+		return fmt.Errorf("failed to install %s: %w", tool.Name, err)
+	}
+
+	// Verify installation
+	ver, checkErr := ti.checkInstalled(*tool)
+	if checkErr != nil {
+		log.Printf("[ToolInstaller] WARNING: %s installed but check failed: %v", tool.Name, checkErr)
+		ti.updateStatus(tool.ID, models.ToolInstalled, "unknown", "")
+	} else {
+		log.Printf("[ToolInstaller] %s installed successfully: %s", tool.Name, ver)
+		ti.updateStatus(tool.ID, models.ToolInstalled, ver, "")
+	}
+
+	return nil
+}
+
+// runInstallScript executes the tool's install script in a subshell.
+func (ti *ToolInstaller) runInstallScript(tool *models.DeployTool) error {
+	// Write script to temp file
+	tmpFile, err := os.CreateTemp("", fmt.Sprintf("tool-install-%s-*.sh", tool.Name))
+	if err != nil {
+		return fmt.Errorf("cannot create temp script: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	version := tool.DefaultVersion
+	if version == "" {
+		version = "latest"
+	}
+
+	script := fmt.Sprintf("#!/bin/sh\nset -e\nTOOL_VERSION=%s\n%s\n", version, tool.InstallScript)
+	if _, err := tmpFile.WriteString(script); err != nil {
+		return fmt.Errorf("cannot write temp script: %w", err)
+	}
+	tmpFile.Close()
+	os.Chmod(tmpFile.Name(), 0755)
+
+	cmd := exec.Command("sh", tmpFile.Name())
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+// downloadBinary downloads a binary file to the install path.
+// Uses multi-mirror fallback: tries each mirror URL in order, with HEAD verification
+// to skip dead mirrors before attempting a full download.
+func (ti *ToolInstaller) downloadBinary(tool *models.DeployTool) error {
+	version := tool.DefaultVersion
+	if version == "" {
+		// Try to get latest version
+		if tool.VersionURL != "" {
+			v, err := ti.fetchVersion(tool.VersionURL)
+			if err != nil {
+				log.Printf("[ToolInstaller] WARNING: cannot fetch version for %s: %v, using default", tool.Name, err)
+			} else {
+				version = v
+			}
+		}
+	}
+	if version == "" {
+		version = "latest"
+	}
+
+	// Resolve mirror URLs — domestic mirrors prioritized when in China
+	mirrorURLs := GetMirrorsForTool(tool.Name, tool.DownloadURL)
+	if len(mirrorURLs) == 0 {
+		return fmt.Errorf("no download URLs configured for %s", tool.Name)
+	}
+
+	// Try each mirror in order
+	var lastErr error
+	for i, urlTemplate := range mirrorURLs {
+		url := strings.ReplaceAll(urlTemplate, "{version}", version)
+		log.Printf("[ToolInstaller] Downloading %s from mirror[%d]: %s", tool.Name, i, url)
+
+		err := ti.tryDownload(url, tool)
+		if err == nil {
+			return nil // success
+		}
+		lastErr = err
+		log.Printf("[ToolInstaller] Mirror[%d] failed for %s: %v, trying next...", i, tool.Name, err)
+	}
+
+	return fmt.Errorf("all %d mirrors exhausted for %s, last error: %w", len(mirrorURLs), tool.Name, lastErr)
+}
+
+// tryDownload attempts a single download from a URL.
+func (ti *ToolInstaller) tryDownload(url string, tool *models.DeployTool) error {
+	// Use a client with timeout for the download phase.
+	// Large files (kubectl ~50MB) can take minutes on slow networks.
+	client := &http.Client{Timeout: 30 * time.Minute}
+	resp, err := client.Get(url)
+	if err != nil {
+		return fmt.Errorf("download failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("download returned HTTP %d", resp.StatusCode)
+	}
+
+	// Write to temp file first, then rename (atomic-ish)
+	installPath := tool.InstallPath
+	if installPath == "" {
+		installPath = filepath.Join("/usr/local/bin", tool.BinaryName)
+	}
+
+	dir := filepath.Dir(installPath)
+	tmpFile, err := os.CreateTemp(dir, fmt.Sprintf(".%s-download-*", tool.Name))
+	if err != nil {
+		return fmt.Errorf("cannot create temp file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	if _, err := io.Copy(tmpFile, resp.Body); err != nil {
+		tmpFile.Close()
+		return fmt.Errorf("failed to save download: %w", err)
+	}
+	tmpFile.Close()
+
+	os.Chmod(tmpFile.Name(), 0755)
+
+	if err := os.Rename(tmpFile.Name(), installPath); err != nil {
+		// Fallback: copy
+		data, _ := os.ReadFile(tmpFile.Name())
+		if writeErr := os.WriteFile(installPath, data, 0755); writeErr != nil {
+			return fmt.Errorf("cannot move binary to %s: %w", installPath, writeErr)
+		}
+	}
+
+	return nil
+}
+
+// fetchVersion gets the latest version string from a URL.
+func (ti *ToolInstaller) fetchVersion(url string) (string, error) {
+	client := &http.Client{Timeout: 15 * time.Second}
+	resp, err := client.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 256))
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(body)), nil
+}
+
+// updateStatus updates the tool's status in the database.
+func (ti *ToolInstaller) updateStatus(toolID uint, status models.ToolStatus, version string, errMsg string) {
+	now := time.Now()
+	updates := map[string]interface{}{
+		"status":    status,
+		"last_check": &now,
+	}
+	if version != "" {
+		updates["installed_version"] = version
+	}
+	if errMsg != "" {
+		updates["error_msg"] = errMsg
+	}
+	ti.db.Model(&models.DeployTool{}).Where("id = ?", toolID).Updates(updates)
+}
+
+// SeedTools inserts predefined tools into the database if the table is empty.
+func (ti *ToolInstaller) SeedTools() {
+	var count int64
+	ti.db.Model(&models.DeployTool{}).Count(&count)
+	if count > 0 {
+		return
+	}
+
+	tools := models.PredefinedTools()
+	now := time.Now()
+	for i := range tools {
+		tools[i].CreatedAt = now
+		tools[i].UpdatedAt = now
+		// Check if preinstalled tools are actually available
+		if tools[i].InstallMethod == models.ToolPreinstall {
+			if ver, err := ti.checkInstalled(tools[i]); err == nil {
+				tools[i].Status = models.ToolInstalled
+				tools[i].InstalledVer = ver
+			} else {
+				tools[i].Status = models.ToolNotInstalled
+			}
+		}
+	}
+
+	if err := ti.db.Create(&tools).Error; err != nil {
+		log.Printf("[ToolInstaller] Seed failed: %v", err)
+	} else {
+		log.Printf("[ToolInstaller] Seeded %d tools", len(tools))
+	}
+}
@@ -0,0 +1,157 @@
+package toolinstaller
+
+import (
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// ============================================
+// 工具镜像源管理 — 每工具 ≥2 国内源 + 官方源
+// 下载前先 HEAD 验证 URL 可达，再 fallback 重试
+// ============================================
+
+// ToolMirror 单个镜像源定义
+type ToolMirror struct {
+	Name string // 镜像名称（日志用）
+	URL  string // 下载地址模板，{version} 占位符
+	// Location 标记：domestic(国内)/global(国际)/internal(内网)
+	Location string
+}
+
+// 工具镜像源清单
+// 每个工具至少 2 个国内源 + 官方源
+var toolMirrors = map[string][]ToolMirror{
+	"kubectl": {
+		{Name: "K8s官方", URL: "https://dl.k8s.io/release/{version}/bin/linux/amd64/kubectl", Location: "global"},
+		{Name: "GCS直链", URL: "https://storage.googleapis.com/kubernetes-release/release/{version}/bin/linux/amd64/kubectl", Location: "global"},
+		// 国内无 kubectl 直链镜像（阿里云/清华只托管 yum/apt 源），此处预留给内网 Harbor
+		// {Name: "内网Harbor", URL: "https://harbor.ops.net/static/kubectl-{version}-linux-amd64", Location: "internal"},
+	},
+	"maven": {
+		{Name: "Apache官方", URL: "https://dlcdn.apache.org/maven/maven-3/{version}/binaries/apache-maven-{version}-bin.tar.gz", Location: "global"},
+		{Name: "清华镜像", URL: "https://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/{version}/binaries/apache-maven-{version}-bin.tar.gz", Location: "domestic"},
+		{Name: "阿里云镜像", URL: "https://mirrors.aliyun.com/apache/maven/maven-3/{version}/binaries/apache-maven-{version}-bin.tar.gz", Location: "domestic"},
+		{Name: "华为云镜像", URL: "https://repo.huaweicloud.com/apache/maven/maven-3/{version}/binaries/apache-maven-{version}-bin.tar.gz", Location: "domestic"},
+	},
+	"nodejs": {
+		{Name: "Node官方", URL: "https://nodejs.org/dist/{version}/node-{version}-linux-x64.tar.xz", Location: "global"},
+		{Name: "npmmirror", URL: "https://npmmirror.com/mirrors/node/{version}/node-{version}-linux-x64.tar.xz", Location: "domestic"},
+		{Name: "清华镜像", URL: "https://mirrors.tuna.tsinghua.edu.cn/nodejs-release/{version}/node-{version}-linux-x64.tar.xz", Location: "domestic"},
+		{Name: "华为云镜像", URL: "https://repo.huaweicloud.com/nodejs/{version}/node-{version}-linux-x64.tar.xz", Location: "domestic"},
+		{Name: "阿里云镜像", URL: "https://mirrors.aliyun.com/nodejs-release/{version}/node-{version}-linux-x64.tar.xz", Location: "domestic"},
+	},
+	"helm": {
+		{Name: "Helm官方", URL: "https://get.helm.sh/helm-{version}-linux-amd64.tar.gz", Location: "global"},
+		{Name: "阿里云镜像", URL: "https://mirrors.aliyun.com/kubernetes/helm/{version}/helm-{version}-linux-amd64.tar.gz", Location: "domestic"},
+	},
+}
+
+// GetMirrorsForTool 返回工具的镜像源列表（按网络环境排序）。
+// 国内网络 → 国内源优先；国际网络 → 官方源优先。
+// 同时过滤掉不可达的镜像。
+func GetMirrorsForTool(toolName, downloadURL string) []string {
+	mirrors, ok := toolMirrors[toolName]
+	if !ok {
+		// 未配置镜像列表的工具，只返回 downloadURL
+		if downloadURL != "" {
+			return []string{downloadURL}
+		}
+		return nil
+	}
+
+	isChina := IsChinaNetwork()
+
+	// 排序：国内网络 → 国内源排前面；国际网络 → 官方源排前面
+	sorted := make([]ToolMirror, len(mirrors))
+	copy(sorted, mirrors)
+
+	// 冒泡排序（简单场景足够）
+	for i := 0; i < len(sorted); i++ {
+		for j := i + 1; j < len(sorted); j++ {
+			// 国内网络：domestic 优先
+			if isChina && sorted[j].Location == "domestic" && sorted[i].Location != "domestic" {
+				sorted[i], sorted[j] = sorted[j], sorted[i]
+			}
+			// 国际网络：global 优先
+			if !isChina && sorted[j].Location == "global" && sorted[i].Location != "global" {
+				sorted[i], sorted[j] = sorted[j], sorted[i]
+			}
+		}
+	}
+
+	// 过滤掉不可达的镜像（HEAD 验证）
+	urls := make([]string, 0, len(sorted))
+	for _, m := range sorted {
+		if verifyURLReachable(m.URL) {
+			urls = append(urls, m.URL)
+			log.Printf("[ToolMirror] ✓ %s (%s) reachable", m.Name, m.URL)
+		} else {
+			log.Printf("[ToolMirror] ✗ %s (%s) unreachable, skip", m.Name, m.URL)
+		}
+	}
+
+	// 如果所有镜像都不可达，仍然返回全部（让下载阶段报错，而非提前过滤掉）
+	if len(urls) == 0 {
+		log.Printf("[ToolMirror] WARNING: all mirrors unreachable for %s, falling back to raw list", toolName)
+		for _, m := range sorted {
+			urls = append(urls, m.URL)
+		}
+	}
+
+	// 把 downloadURL 也作为兜底放在最后（如果不在 mirrors 中）
+	if downloadURL != "" && !containsURL(urls, downloadURL) {
+		urls = append(urls, downloadURL)
+	}
+
+	return urls
+}
+
+// verifyURLReachable 通过 HEAD 请求（带重定向跟随）验证 URL 是否可达。
+// 返回 true 表示 URL 可访问（HTTP 2xx/3xx）。
+func verifyURLReachable(urlTemplate string) bool {
+	// 对 URL 模板做简单替换，用已知存在的版本测试
+	testURL := strings.ReplaceAll(urlTemplate, "{version}", "v1.32.0")
+	testURL = strings.ReplaceAll(testURL, "{version}", "3.9.9") // maven 版本格式不同
+
+	client := &http.Client{
+		Timeout: 5 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= 3 {
+				return http.ErrUseLastResponse
+			}
+			return nil
+		},
+	}
+
+	resp, err := client.Head(testURL)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	// 2xx 或 3xx 都算可达
+	return resp.StatusCode >= 200 && resp.StatusCode < 400
+}
+
+func containsURL(urls []string, target string) bool {
+	for _, u := range urls {
+		if u == target {
+			return true
+		}
+	}
+	return false
+}
+
+// VerifyAndGetBestMirror 综合检测：先 HEAD 测试所有镜像，返回第一个可达的。
+// 如果全不可达，返回空字符串。
+func VerifyAndGetBestMirror(toolName string, mirrors []string) string {
+	for _, url := range mirrors {
+		if verifyURLReachable(url) {
+			return url
+		}
+	}
+	log.Printf("[ToolMirror] WARNING: no reachable mirror found for %s", toolName)
+	return ""
+}
@@ -0,0 +1,99 @@
+package toolinstaller
+
+import (
+	"net/http"
+	"sync"
+	"time"
+)
+
+// ============================================
+// 网络环境检测 — 多端点交叉验证判断是否在中国网络内
+// 原理：Google/YouTube 在中国被墙，阿里云/清华镜像仅国内可达
+// 4 条线路 ≥3 条一致才判定，防止单点故障误判
+// ============================================
+
+// chinaEndpoint 检测端点定义
+type chinaEndpoint struct {
+	URL     string        // 检测 URL
+	Name    string        // 端点名称（日志用）
+	Timeout time.Duration // 超时时间
+	// 在中国网络内的预期可达性
+	ReachableInChina bool
+}
+
+var chinaEndpoints = []chinaEndpoint{
+	{URL: "https://www.google.com", Name: "Google", Timeout: 2 * time.Second, ReachableInChina: false},
+	{URL: "https://www.youtube.com", Name: "YouTube", Timeout: 2 * time.Second, ReachableInChina: false},
+	{URL: "https://www.baidu.com", Name: "Baidu", Timeout: 2 * time.Second, ReachableInChina: true},
+	{URL: "https://mirrors.aliyun.com", Name: "Aliyun", Timeout: 3 * time.Second, ReachableInChina: true},
+	{URL: "https://mirrors.tuna.tsinghua.edu.cn", Name: "Tsinghua", Timeout: 3 * time.Second, ReachableInChina: true},
+}
+
+var (
+	networkChecked bool
+	networkMu      sync.Mutex
+	isChina        bool
+)
+
+// IsChinaNetwork 检测当前是否在中国网络环境。
+// 使用 5 个端点交叉验证：Google/YouTube 不可达 + 百度/阿里云/清华可达 → 判定为国内。
+// 检测结果缓存，后续调用直接返回缓存值。
+func IsChinaNetwork() bool {
+	networkMu.Lock()
+	defer networkMu.Unlock()
+
+	if networkChecked {
+		return isChina
+	}
+
+	// 并发检测所有端点
+	type result struct {
+		name      string
+		reachable bool
+	}
+	results := make([]result, len(chinaEndpoints))
+	var wg sync.WaitGroup
+
+	for i, ep := range chinaEndpoints {
+		wg.Add(1)
+		go func(idx int, endpoint chinaEndpoint) {
+			defer wg.Done()
+			client := &http.Client{Timeout: endpoint.Timeout}
+			resp, err := client.Head(endpoint.URL)
+			reachable := err == nil && resp != nil && resp.StatusCode < 500
+			if resp != nil {
+				resp.Body.Close()
+			}
+			results[idx] = result{name: endpoint.Name, reachable: reachable}
+		}(i, ep)
+	}
+	wg.Wait()
+
+	// 统计：预期不可达（国外）vs 预期可达（国内）
+	var foreignUnreachable, domesticReachable int
+	for i, r := range results {
+		ep := chinaEndpoints[i]
+		if ep.ReachableInChina {
+			if r.reachable {
+				domesticReachable++
+			}
+		} else {
+			if !r.reachable {
+				foreignUnreachable++
+			}
+		}
+	}
+
+	// 判定逻辑：国外端点全部不可达 && 国内端点至少 2 个可达
+	isChina = foreignUnreachable >= 2 && domesticReachable >= 2
+	networkChecked = true
+	return isChina
+}
+
+// ResetNetworkCheck 重置网络检测缓存（用于网络环境变化后重新检测）
+func ResetNetworkCheck() {
+	networkMu.Lock()
+	defer networkMu.Unlock()
+	networkChecked = false
+	isChina = false
+}