fix: 恢复腾讯云账单同步并完善成本预测
- 接入腾讯云单日账单、全局限频和指数退避 - 修正月末费用预测并增加账期提示和费用归因 - 补充腾讯云限频及账单归因回归测试
This commit is contained in:
Logan.Li committed
1 parent
88c41f0b2b
commit
dd3770a66c
15 files changed
+610
-17
No files matched your search
@@ -16,13 +16,16 @@ type BillingFetcher interface {
|
||||
FetchDailyBill(billingDate string) ([]CloudBillingRecord, error)
|
||||
}
|
||||
|
||||
// SupportedBillingProviders 已接入账单 API 的云厂商列表。 // Logan.Li
|
||||
var SupportedBillingProviders = []string{"aliyun", "tencent"}
|
||||
|
||||
// NewBillingFetcher 根据云厂商创建账单拉取器
|
||||
func NewBillingFetcher(provider, accessKey, secretKey string) BillingFetcher {
|
||||
switch provider {
|
||||
case "aliyun":
|
||||
if p := NewAliyunProvider(accessKey, secretKey); p != nil {
|
||||
return p
|
||||
}
|
||||
return NewAliyunProvider(accessKey, secretKey)
|
||||
case "tencent":
|
||||
return NewTencentProvider(accessKey, secretKey)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -5,17 +5,39 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TencentProvider 通过腾讯云 TC3-HMAC-SHA256 API 同步 CVM 资源。 // Logan.Li
|
||||
type TencentProvider struct{ accessKey, secretKey string }
|
||||
|
||||
const (
|
||||
tencentRequestInterval = 260 * time.Millisecond
|
||||
tencentMaxAttempts = 4
|
||||
)
|
||||
|
||||
var tencentLimiter = struct {
|
||||
sync.Mutex
|
||||
next time.Time
|
||||
}{}
|
||||
|
||||
// tencentAPIError 保留腾讯云错误码，便于对限频错误做退避重试。 // Logan.Li
|
||||
type tencentAPIError struct {
|
||||
Code string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *tencentAPIError) Error() string {
|
||||
return fmt.Sprintf("腾讯云 API 错误 %s: %s", e.Code, e.Message)
|
||||
}
|
||||
|
||||
// tencentInstancesResponse 描述腾讯云 CVM 实例分页响应。 // Logan.Li
|
||||
type tencentInstancesResponse struct {
|
||||
Response struct {
|
||||
@@ -58,6 +80,44 @@ func (p *TencentProvider) callInRegion(region, action string, payload interface{
|
||||
|
||||
// callService 使用 TC3-HMAC-SHA256 调用腾讯云服务 API。 // Logan.Li
|
||||
func (p *TencentProvider) callService(service, host, version, action, region string, payload interface{}) ([]byte, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < tencentMaxAttempts; attempt++ {
|
||||
waitTencentRequestSlot()
|
||||
data, err := p.callServiceOnce(service, host, version, action, region, payload)
|
||||
if err == nil {
|
||||
return data, nil
|
||||
}
|
||||
lastErr = err
|
||||
delay, retry := tencentRetryDelay(err, attempt)
|
||||
if !retry {
|
||||
return nil, err
|
||||
}
|
||||
time.Sleep(delay)
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// waitTencentRequestSlot 将进程内所有腾讯云请求统一限速在每秒 4 次以内。 // Logan.Li
|
||||
func waitTencentRequestSlot() {
|
||||
tencentLimiter.Lock()
|
||||
defer tencentLimiter.Unlock()
|
||||
if delay := time.Until(tencentLimiter.next); delay > 0 {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
tencentLimiter.next = time.Now().Add(tencentRequestInterval)
|
||||
}
|
||||
|
||||
// tencentRetryDelay 对 RequestLimitExceeded 使用指数退避，其他错误立即返回。 // Logan.Li
|
||||
func tencentRetryDelay(err error, attempt int) (time.Duration, bool) {
|
||||
var apiErr *tencentAPIError
|
||||
if !errors.As(err, &apiErr) || !strings.HasPrefix(apiErr.Code, "RequestLimitExceeded") || attempt >= tencentMaxAttempts-1 {
|
||||
return 0, false
|
||||
}
|
||||
return (500 * time.Millisecond) * time.Duration(1<<attempt), true
|
||||
}
|
||||
|
||||
// callServiceOnce 执行一次腾讯云 TC3-HMAC-SHA256 签名请求。 // Logan.Li
|
||||
func (p *TencentProvider) callServiceOnce(service, host, version, action, region string, payload interface{}) ([]byte, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("编码腾讯云请求失败: %w", err)
|
||||
@@ -95,6 +155,9 @@ func (p *TencentProvider) callService(service, host, version, action, region str
|
||||
return nil, fmt.Errorf("读取腾讯云响应失败: %w", err)
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
return nil, &tencentAPIError{Code: "RequestLimitExceeded.HTTP429", Message: string(data)}
|
||||
}
|
||||
return nil, fmt.Errorf("腾讯云 API 返回 HTTP %d: %s", resp.StatusCode, string(data))
|
||||
}
|
||||
var envelope struct {
|
||||
@@ -106,7 +169,7 @@ func (p *TencentProvider) callService(service, host, version, action, region str
|
||||
} `json:"Response"`
|
||||
}
|
||||
if json.Unmarshal(data, &envelope) == nil && envelope.Response.Error != nil {
|
||||
return nil, fmt.Errorf("腾讯云 API 错误 %s: %s", envelope.Response.Error.Code, envelope.Response.Error.Message)
|
||||
return nil, &tencentAPIError{Code: envelope.Response.Error.Code, Message: envelope.Response.Error.Message}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FetchDailyBill 拉取指定日期的腾讯云日账单，并按产品和地域汇总。 // Logan.Li
|
||||
func (p *TencentProvider) FetchDailyBill(billingDate string) ([]CloudBillingRecord, error) {
|
||||
if len(billingDate) != 10 {
|
||||
return nil, fmt.Errorf("账单日期格式应为 YYYY-MM-DD")
|
||||
}
|
||||
|
||||
type aggregationKey struct {
|
||||
service string
|
||||
region string
|
||||
}
|
||||
aggregation := map[aggregationKey]float64{}
|
||||
const pageSize = 100
|
||||
|
||||
for offset := 0; ; offset += pageSize {
|
||||
// DescribeBillDetail 不支持 Day 参数，使用账期和起止时间限定单日范围。 // Logan.Li
|
||||
body, err := p.callService("billing", "billing.tencentcloudapi.com", "2018-07-09", "DescribeBillDetail", "", map[string]interface{}{
|
||||
"Month": billingDate[:7],
|
||||
"BeginTime": billingDate + " 00:00:00",
|
||||
"EndTime": billingDate + " 23:59:59",
|
||||
"Offset": offset,
|
||||
"Limit": pageSize,
|
||||
"NeedRecordNum": 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response tencentBillDetailResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("解析腾讯云账单响应失败: %w", err)
|
||||
}
|
||||
for _, detail := range response.Response.DetailSet {
|
||||
cost := tencentBillAmount(detail.RealTotalCost)
|
||||
if cost == 0 {
|
||||
cost = tencentBillAmount(detail.RealCost)
|
||||
}
|
||||
if cost == 0 {
|
||||
cost = tencentBillAmount(detail.CashPayAmount)
|
||||
}
|
||||
if cost == 0 {
|
||||
// 腾讯云明细账单将组件费用放在 ComponentSet，逐项汇总实际费用。 // Logan.Li
|
||||
for _, component := range detail.ComponentSet {
|
||||
componentCost := tencentBillAmount(component.RealCost)
|
||||
if componentCost == 0 {
|
||||
componentCost = tencentBillAmount(component.CashPayAmount)
|
||||
}
|
||||
cost += componentCost
|
||||
}
|
||||
}
|
||||
if cost <= 0 {
|
||||
continue
|
||||
}
|
||||
service := firstNonEmpty(detail.ProductCodeName, detail.BusinessCodeName, detail.ProductCode, "其他")
|
||||
region := firstNonEmpty(detail.RegionName, "全局")
|
||||
aggregation[aggregationKey{service: service, region: region}] += cost
|
||||
}
|
||||
|
||||
if len(response.Response.DetailSet) < pageSize || (response.Response.Total > 0 && offset+pageSize >= response.Response.Total) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]aggregationKey, 0, len(aggregation))
|
||||
for key := range aggregation {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].service == keys[j].service {
|
||||
return keys[i].region < keys[j].region
|
||||
}
|
||||
return keys[i].service < keys[j].service
|
||||
})
|
||||
|
||||
records := make([]CloudBillingRecord, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
records = append(records, CloudBillingRecord{
|
||||
BillingDate: billingDate,
|
||||
Provider: "tencent",
|
||||
Cost: math.Round(aggregation[key]*100) / 100,
|
||||
Currency: "CNY",
|
||||
Service: key.service,
|
||||
Region: key.region,
|
||||
Description: "云 API 自动同步（腾讯云 DescribeBillDetail）",
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// tencentBillDetailResponse 描述腾讯云费用中心明细账单响应。 // Logan.Li
|
||||
type tencentBillDetailResponse struct {
|
||||
Response struct {
|
||||
Total int `json:"Total"`
|
||||
DetailSet []struct {
|
||||
ProductCode string `json:"ProductCode"`
|
||||
ProductCodeName string `json:"ProductCodeName"`
|
||||
BusinessCodeName string `json:"BusinessCodeName"`
|
||||
RegionName string `json:"RegionName"`
|
||||
RealTotalCost json.RawMessage `json:"RealTotalCost"`
|
||||
RealCost json.RawMessage `json:"RealCost"`
|
||||
CashPayAmount json.RawMessage `json:"CashPayAmount"`
|
||||
ComponentSet []struct {
|
||||
RealCost json.RawMessage `json:"RealCost"`
|
||||
CashPayAmount json.RawMessage `json:"CashPayAmount"`
|
||||
} `json:"ComponentSet"`
|
||||
} `json:"DetailSet"`
|
||||
} `json:"Response"`
|
||||
}
|
||||
|
||||
// tencentBillAmount 兼容腾讯云账单金额字段为字符串或数字的响应。 // Logan.Li
|
||||
func tencentBillAmount(raw json.RawMessage) float64 {
|
||||
value := strings.Trim(strings.TrimSpace(string(raw)), "\"")
|
||||
amount, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return amount
|
||||
}
|
||||
|
||||
// firstNonEmpty 返回首个非空文本，用于兼容不同账单产品字段。 // Logan.Li
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if text := strings.TrimSpace(value); text != "" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestTencentRetryDelay 验证限频错误指数退避与最大重试次数。 // Logan.Li
|
||||
func TestTencentRetryDelay(t *testing.T) {
|
||||
rateLimitErr := &tencentAPIError{Code: "RequestLimitExceeded", Message: "frequency limit"}
|
||||
delay, retry := tencentRetryDelay(rateLimitErr, 0)
|
||||
if !retry || delay != 500*time.Millisecond {
|
||||
t.Fatalf("首次限频退避错误: retry=%v delay=%s", retry, delay)
|
||||
}
|
||||
|
||||
delay, retry = tencentRetryDelay(rateLimitErr, 2)
|
||||
if !retry || delay != 2*time.Second {
|
||||
t.Fatalf("第三次限频退避错误: retry=%v delay=%s", retry, delay)
|
||||
}
|
||||
|
||||
if _, retry = tencentRetryDelay(rateLimitErr, tencentMaxAttempts-1); retry {
|
||||
t.Fatal("达到最大次数后不应继续重试")
|
||||
}
|
||||
if _, retry = tencentRetryDelay(errors.New("鉴权失败"), 0); retry {
|
||||
t.Fatal("非限频错误不应重试")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user