fix: 恢复腾讯云账单同步并完善成本预测

- 接入腾讯云单日账单、全局限频和指数退避

- 修正月末费用预测并增加账期提示和费用归因

- 补充腾讯云限频及账单归因回归测试
This commit is contained in:
Logan.Li committed 2026-07-29 05:44:40 +08:00
1 parent 88c41f0b2b
commit dd3770a66c
15 files changed
+610 -17

No files matched your search

+7 -4
View File
@@ -16,13 +16,16 @@ type BillingFetcher interface {
FetchDailyBill(billingDate string) ([]CloudBillingRecord, error)
}
// SupportedBillingProviders 已接入账单 API 的云厂商列表。 // Logan.Li
var SupportedBillingProviders = []string{"aliyun", "tencent"}
// NewBillingFetcher 根据云厂商创建账单拉取器
func NewBillingFetcher(provider, accessKey, secretKey string) BillingFetcher {
switch provider {
case "aliyun":
if p := NewAliyunProvider(accessKey, secretKey); p != nil {
return p
}
return NewAliyunProvider(accessKey, secretKey)
case "tencent":
return NewTencentProvider(accessKey, secretKey)
}
return nil
}
}
+64 -1
View File
@@ -5,17 +5,39 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"strings"
"sync"
"time"
)
// TencentProvider 通过腾讯云 TC3-HMAC-SHA256 API 同步 CVM 资源。 // Logan.Li
type TencentProvider struct{ accessKey, secretKey string }
const (
tencentRequestInterval = 260 * time.Millisecond
tencentMaxAttempts = 4
)
var tencentLimiter = struct {
sync.Mutex
next time.Time
}{}
// tencentAPIError 保留腾讯云错误码,便于对限频错误做退避重试。 // Logan.Li
type tencentAPIError struct {
Code string
Message string
}
func (e *tencentAPIError) Error() string {
return fmt.Sprintf("腾讯云 API 错误 %s: %s", e.Code, e.Message)
}
// tencentInstancesResponse 描述腾讯云 CVM 实例分页响应。 // Logan.Li
type tencentInstancesResponse struct {
Response struct {
@@ -58,6 +80,44 @@ func (p *TencentProvider) callInRegion(region, action string, payload interface{
// callService 使用 TC3-HMAC-SHA256 调用腾讯云服务 API。 // Logan.Li
func (p *TencentProvider) callService(service, host, version, action, region string, payload interface{}) ([]byte, error) {
var lastErr error
for attempt := 0; attempt < tencentMaxAttempts; attempt++ {
waitTencentRequestSlot()
data, err := p.callServiceOnce(service, host, version, action, region, payload)
if err == nil {
return data, nil
}
lastErr = err
delay, retry := tencentRetryDelay(err, attempt)
if !retry {
return nil, err
}
time.Sleep(delay)
}
return nil, lastErr
}
// waitTencentRequestSlot 将进程内所有腾讯云请求统一限速在每秒 4 次以内。 // Logan.Li
func waitTencentRequestSlot() {
tencentLimiter.Lock()
defer tencentLimiter.Unlock()
if delay := time.Until(tencentLimiter.next); delay > 0 {
time.Sleep(delay)
}
tencentLimiter.next = time.Now().Add(tencentRequestInterval)
}
// tencentRetryDelay 对 RequestLimitExceeded 使用指数退避,其他错误立即返回。 // Logan.Li
func tencentRetryDelay(err error, attempt int) (time.Duration, bool) {
var apiErr *tencentAPIError
if !errors.As(err, &apiErr) || !strings.HasPrefix(apiErr.Code, "RequestLimitExceeded") || attempt >= tencentMaxAttempts-1 {
return 0, false
}
return (500 * time.Millisecond) * time.Duration(1<<attempt), true
}
// callServiceOnce 执行一次腾讯云 TC3-HMAC-SHA256 签名请求。 // Logan.Li
func (p *TencentProvider) callServiceOnce(service, host, version, action, region string, payload interface{}) ([]byte, error) {
body, err := json.Marshal(payload)
if err != nil {
return nil, fmt.Errorf("编码腾讯云请求失败: %w", err)
@@ -95,6 +155,9 @@ func (p *TencentProvider) callService(service, host, version, action, region str
return nil, fmt.Errorf("读取腾讯云响应失败: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
if resp.StatusCode == http.StatusTooManyRequests {
return nil, &tencentAPIError{Code: "RequestLimitExceeded.HTTP429", Message: string(data)}
}
return nil, fmt.Errorf("腾讯云 API 返回 HTTP %d: %s", resp.StatusCode, string(data))
}
var envelope struct {
@@ -106,7 +169,7 @@ func (p *TencentProvider) callService(service, host, version, action, region str
} `json:"Response"`
}
if json.Unmarshal(data, &envelope) == nil && envelope.Response.Error != nil {
return nil, fmt.Errorf("腾讯云 API 错误 %s: %s", envelope.Response.Error.Code, envelope.Response.Error.Message)
return nil, &tencentAPIError{Code: envelope.Response.Error.Code, Message: envelope.Response.Error.Message}
}
return data, nil
}
+138
View File
@@ -0,0 +1,138 @@
package cloud
import (
"encoding/json"
"fmt"
"math"
"sort"
"strconv"
"strings"
)
// FetchDailyBill 拉取指定日期的腾讯云日账单,并按产品和地域汇总。 // Logan.Li
func (p *TencentProvider) FetchDailyBill(billingDate string) ([]CloudBillingRecord, error) {
if len(billingDate) != 10 {
return nil, fmt.Errorf("账单日期格式应为 YYYY-MM-DD")
}
type aggregationKey struct {
service string
region string
}
aggregation := map[aggregationKey]float64{}
const pageSize = 100
for offset := 0; ; offset += pageSize {
// DescribeBillDetail 不支持 Day 参数,使用账期和起止时间限定单日范围。 // Logan.Li
body, err := p.callService("billing", "billing.tencentcloudapi.com", "2018-07-09", "DescribeBillDetail", "", map[string]interface{}{
"Month": billingDate[:7],
"BeginTime": billingDate + " 00:00:00",
"EndTime": billingDate + " 23:59:59",
"Offset": offset,
"Limit": pageSize,
"NeedRecordNum": 1,
})
if err != nil {
return nil, err
}
var response tencentBillDetailResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("解析腾讯云账单响应失败: %w", err)
}
for _, detail := range response.Response.DetailSet {
cost := tencentBillAmount(detail.RealTotalCost)
if cost == 0 {
cost = tencentBillAmount(detail.RealCost)
}
if cost == 0 {
cost = tencentBillAmount(detail.CashPayAmount)
}
if cost == 0 {
// 腾讯云明细账单将组件费用放在 ComponentSet,逐项汇总实际费用。 // Logan.Li
for _, component := range detail.ComponentSet {
componentCost := tencentBillAmount(component.RealCost)
if componentCost == 0 {
componentCost = tencentBillAmount(component.CashPayAmount)
}
cost += componentCost
}
}
if cost <= 0 {
continue
}
service := firstNonEmpty(detail.ProductCodeName, detail.BusinessCodeName, detail.ProductCode, "其他")
region := firstNonEmpty(detail.RegionName, "全局")
aggregation[aggregationKey{service: service, region: region}] += cost
}
if len(response.Response.DetailSet) < pageSize || (response.Response.Total > 0 && offset+pageSize >= response.Response.Total) {
break
}
}
keys := make([]aggregationKey, 0, len(aggregation))
for key := range aggregation {
keys = append(keys, key)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].service == keys[j].service {
return keys[i].region < keys[j].region
}
return keys[i].service < keys[j].service
})
records := make([]CloudBillingRecord, 0, len(keys))
for _, key := range keys {
records = append(records, CloudBillingRecord{
BillingDate: billingDate,
Provider: "tencent",
Cost: math.Round(aggregation[key]*100) / 100,
Currency: "CNY",
Service: key.service,
Region: key.region,
Description: "云 API 自动同步(腾讯云 DescribeBillDetail)",
})
}
return records, nil
}
// tencentBillDetailResponse 描述腾讯云费用中心明细账单响应。 // Logan.Li
type tencentBillDetailResponse struct {
Response struct {
Total int `json:"Total"`
DetailSet []struct {
ProductCode string `json:"ProductCode"`
ProductCodeName string `json:"ProductCodeName"`
BusinessCodeName string `json:"BusinessCodeName"`
RegionName string `json:"RegionName"`
RealTotalCost json.RawMessage `json:"RealTotalCost"`
RealCost json.RawMessage `json:"RealCost"`
CashPayAmount json.RawMessage `json:"CashPayAmount"`
ComponentSet []struct {
RealCost json.RawMessage `json:"RealCost"`
CashPayAmount json.RawMessage `json:"CashPayAmount"`
} `json:"ComponentSet"`
} `json:"DetailSet"`
} `json:"Response"`
}
// tencentBillAmount 兼容腾讯云账单金额字段为字符串或数字的响应。 // Logan.Li
func tencentBillAmount(raw json.RawMessage) float64 {
value := strings.Trim(strings.TrimSpace(string(raw)), "\"")
amount, err := strconv.ParseFloat(value, 64)
if err != nil {
return 0
}
return amount
}
// firstNonEmpty 返回首个非空文本,用于兼容不同账单产品字段。 // Logan.Li
func firstNonEmpty(values ...string) string {
for _, value := range values {
if text := strings.TrimSpace(value); text != "" {
return text
}
}
return ""
}
+28
View File
@@ -0,0 +1,28 @@
package cloud
import (
"errors"
"testing"
"time"
)
// TestTencentRetryDelay 验证限频错误指数退避与最大重试次数。 // Logan.Li
func TestTencentRetryDelay(t *testing.T) {
rateLimitErr := &tencentAPIError{Code: "RequestLimitExceeded", Message: "frequency limit"}
delay, retry := tencentRetryDelay(rateLimitErr, 0)
if !retry || delay != 500*time.Millisecond {
t.Fatalf("首次限频退避错误: retry=%v delay=%s", retry, delay)
}
delay, retry = tencentRetryDelay(rateLimitErr, 2)
if !retry || delay != 2*time.Second {
t.Fatalf("第三次限频退避错误: retry=%v delay=%s", retry, delay)
}
if _, retry = tencentRetryDelay(rateLimitErr, tencentMaxAttempts-1); retry {
t.Fatal("达到最大次数后不应继续重试")
}
if _, retry = tencentRetryDelay(errors.New("鉴权失败"), 0); retry {
t.Fatal("非限频错误不应重试")
}
}
+160
View File
@@ -0,0 +1,160 @@
package controller
import (
"fmt"
"math"
"net/http"
"sort"
"time"
"ezops/model"
"github.com/gin-gonic/gin"
)
// billingMonthCost 表示月度费用汇总。 // Logan.Li
type billingMonthCost struct {
Month string `json:"month"`
Cost float64 `json:"cost"`
}
// billingAttribution 表示可复核的费用变化归因。 // Logan.Li
type billingAttribution struct {
Dimension string `json:"dimension"`
Name string `json:"name"`
BaselineCost float64 `json:"baseline_cost"`
CurrentCost float64 `json:"current_cost"`
Delta float64 `json:"delta"`
Contribution float64 `json:"contribution"`
}
// GetBillingCostAnalysis 返回月度趋势和同期费用归因,AI 只负责解释确定性结果。 // Logan.Li
func (h *BillingHandler) GetBillingCostAnalysis(c *gin.Context) {
now := time.Now()
currentStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
baselineStart := currentStart.AddDate(0, -1, 0)
baselineEndDay := now.Day()
baselineLastDay := baselineStart.AddDate(0, 1, -1).Day()
if baselineEndDay > baselineLastDay {
baselineEndDay = baselineLastDay
}
baselineEnd := time.Date(baselineStart.Year(), baselineStart.Month(), baselineEndDay, 23, 59, 59, 0, now.Location())
analysisStart := currentStart.AddDate(0, -5, 0)
currency := c.DefaultQuery("currency", "CNY")
query := h.db.Model(&model.CloudBilling{}).Where("billing_date >= ? AND currency = ?", analysisStart.Format("2006-01-02"), currency)
type monthRow struct {
Month string
Cost float64
}
var monthRows []monthRow
query.Select("SUBSTR(billing_date, 1, 7) AS month, COALESCE(SUM(cost), 0) AS cost").
Group("SUBSTR(billing_date, 1, 7)").Order("month ASC").Scan(&monthRows)
months := make([]billingMonthCost, 0, len(monthRows))
for _, row := range monthRows {
months = append(months, billingMonthCost{Month: row.Month, Cost: roundBilling(row.Cost)})
}
baselineStartDate := baselineStart.Format("2006-01-02")
baselineEndDate := baselineEnd.Format("2006-01-02")
currentStartDate := currentStart.Format("2006-01-02")
currentEndDate := now.Format("2006-01-02")
baselineCost := h.sumBillingPeriod(baselineStartDate, baselineEndDate, currency)
currentCost := h.sumBillingPeriod(currentStartDate, currentEndDate, currency)
totalDelta := currentCost - baselineCost
attributions := make([]billingAttribution, 0, 24)
for _, dimension := range []string{"provider", "account_name", "service", "region"} {
attributions = append(attributions, h.billingDimensionAttribution(
dimension, baselineStartDate, baselineEndDate, currentStartDate, currentEndDate, currency, totalDelta,
)...)
}
sort.Slice(attributions, func(i, j int) bool {
return math.Abs(attributions[i].Delta) > math.Abs(attributions[j].Delta)
})
if len(attributions) > 20 {
attributions = attributions[:20]
}
var dataThrough string
h.db.Model(&model.CloudBilling{}).Where("currency = ?", currency).Select("COALESCE(MAX(billing_date), '')").Scan(&dataThrough)
c.JSON(http.StatusOK, model.Response{Code: 200, Message: "success", Data: gin.H{
"analysis_start": analysisStart.Format("2006-01-02"),
"analysis_end": currentEndDate,
"data_through": dataThrough,
"currency": currency,
"incomplete_period": true,
"months": months,
"baseline_period": fmt.Sprintf("%s - %s", baselineStartDate, baselineEndDate),
"comparison_period": fmt.Sprintf("%s - %s", currentStartDate, currentEndDate),
"baseline_cost": roundBilling(baselineCost),
"comparison_cost": roundBilling(currentCost),
"total_delta": roundBilling(totalDelta),
"attributions": attributions,
"contribution_formula": "(当前期费用 - 基准期费用) / (当前期总费用 - 基准期总费用)",
}})
}
// sumBillingPeriod 汇总指定币种和日期范围费用。 // Logan.Li
func (h *BillingHandler) sumBillingPeriod(startDate, endDate, currency string) float64 {
var total float64
h.db.Model(&model.CloudBilling{}).
Where("billing_date BETWEEN ? AND ? AND currency = ?", startDate, endDate, currency).
Select("COALESCE(SUM(cost), 0)").Scan(&total)
return total
}
// billingDimensionAttribution 计算单一维度的同期费用变化贡献。 // Logan.Li
func (h *BillingHandler) billingDimensionAttribution(dimension, baselineStart, baselineEnd, currentStart, currentEnd, currency string, totalDelta float64) []billingAttribution {
type dimensionRow struct {
Name string
Cost float64
}
load := func(startDate, endDate string) []dimensionRow {
var rows []dimensionRow
h.db.Model(&model.CloudBilling{}).
Select(dimension+" AS name, COALESCE(SUM(cost), 0) AS cost").
Where("billing_date BETWEEN ? AND ? AND currency = ?", startDate, endDate, currency).
Group(dimension).Scan(&rows)
return rows
}
baseline := map[string]float64{}
current := map[string]float64{}
for _, row := range load(baselineStart, baselineEnd) {
baseline[row.Name] = row.Cost
}
for _, row := range load(currentStart, currentEnd) {
current[row.Name] = row.Cost
}
for name := range baseline {
if _, ok := current[name]; !ok {
current[name] = 0
}
}
result := make([]billingAttribution, 0, len(current))
for rawName, currentCost := range current {
name := rawName
if rawName == "" {
name = "未分类"
}
baselineCost := baseline[rawName]
delta := currentCost - baselineCost
if math.Abs(delta) < 0.01 {
continue
}
contribution := 0.0
if math.Abs(totalDelta) >= 0.01 {
contribution = delta / totalDelta * 100
}
result = append(result, billingAttribution{
Dimension: dimension, Name: name, BaselineCost: roundBilling(baselineCost),
CurrentCost: roundBilling(currentCost), Delta: roundBilling(delta), Contribution: roundBilling(contribution),
})
}
return result
}
// roundBilling 统一费用和百分比精度。 // Logan.Li
func roundBilling(value float64) float64 {
return math.Round(value*100) / 100
}
@@ -0,0 +1,45 @@
package controller
import (
"testing"
"ezops/model"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
// TestBillingDimensionAttribution 验证费用变化额和贡献度使用数据库完整数据计算。 // Logan.Li
func TestBillingDimensionAttribution(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatalf("打开测试数据库失败: %v", err)
}
if err = db.AutoMigrate(&model.CloudBilling{}); err != nil {
t.Fatalf("迁移账单表失败: %v", err)
}
records := []model.CloudBilling{
{BillingDate: "2026-06-01", Provider: "tencent", Service: "CVM", Region: "广州", AccountName: "预发", Currency: "CNY", Cost: 100},
{BillingDate: "2026-07-01", Provider: "tencent", Service: "CVM", Region: "广州", AccountName: "预发", Currency: "CNY", Cost: 180},
{BillingDate: "2026-07-01", Provider: "tencent", Service: "COS", Region: "广州", AccountName: "预发", Currency: "CNY", Cost: 20},
}
if err = db.Create(&records).Error; err != nil {
t.Fatalf("写入测试账单失败: %v", err)
}
handler := NewBillingHandler(db)
items := handler.billingDimensionAttribution("service", "2026-06-01", "2026-06-30", "2026-07-01", "2026-07-31", "CNY", 100)
if len(items) != 2 {
t.Fatalf("归因项数量错误: got=%d want=2", len(items))
}
result := map[string]billingAttribution{}
for _, item := range items {
result[item.Name] = item
}
if result["CVM"].Delta != 80 || result["CVM"].Contribution != 80 {
t.Fatalf("CVM 归因错误: %+v", result["CVM"])
}
if result["COS"].Delta != 20 || result["COS"].Contribution != 20 {
t.Fatalf("COS 归因错误: %+v", result["COS"])
}
}
+4 -1
View File
@@ -37,6 +37,9 @@ func SyncDate(db *gorm.DB, billingDate string, accountID *uint) Result {
q := db.Where("status = ?", "active")
if accountID != nil {
q = q.Where("id = ?", *accountID)
} else {
// 批量同步只选择已接入账单 API 的厂商,避免无效失败。 // Logan.Li
q = q.Where("provider IN ?", cloud.SupportedBillingProviders)
}
q.Find(&accounts)
result.Accounts = len(accounts)
@@ -138,4 +141,4 @@ func syncAccount(db *gorm.DB, account model.CloudAccount, billingDate string) (c
db.Model(&account).Update("monthly_cost", monthCost)
return created, updated, skipped, nil
}
}
+1
View File
@@ -323,6 +323,7 @@ func Register(r *gin.Engine, db *gorm.DB, log *zap.Logger, cfg *config.Config, c
billings.POST("/batch", billingHandler.BatchCreateBillings)
billings.DELETE("/:id", billingHandler.DeleteBilling)
billings.GET("/stats", billingHandler.GetBillingStats)
billings.GET("/cost-analysis", billingHandler.GetBillingCostAnalysis)
billings.GET("/providers", billingHandler.GetProviderOptions)
billings.POST("/sync", billingHandler.SyncYesterdayBillings)
}