rlwj-lab/opsxsql
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 数据脱敏体系重构 — 全局设置+masked_columns+前端去硬编码

Browse Source 
后端:
- applyDataMasking 新增 masked_columns 返回字段，告诉前端哪些列被脱敏
- seedDefaultSettings 添加 data_masking 分组(启用开关+敏感字段列表)
- 脱敏逻辑: 全局敏感字段 + 实例级 mask_fields 合并，开关关闭则跳过

前端:
- QueriesPage 去掉硬编码 sensitiveKeywords 和 maskData()
- 改用后端返回的 masked_columns 显示'敏'标签
- 脱敏值由后端统一处理(替换为*)
- PlatformConfigPage 数据脱敏分组独立显示
- EditInstanceDialog 脱敏字段简化为逗号分隔输入

验证:
- 设置页开关可正常开关/保存
- 查询 ghost_test: email列被脱敏为*，列头显示'敏'标签
- 非敏感列(name, id, status)正常显示
This commit is contained in:
logan committed 2026-05-28 23:26:10 +08:00
1 parent e81eed288c
commit 557d88d3ed
6 files changed
+155 -62
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/cmd/server/main.go
+21
View File
@@ -60,6 +60,9 @@ func main() {
&model.UserGroup{},
)
// Seed default settings (idempotent)
seedDefaultSettings(db)
// Setup Gin
r := gin.Default()
r.Use(middleware.CORSMiddleware())
@@ -79,3 +82,21 @@ func main() {
log.Fatalf("Failed to start server: %v", err)
}
}
func seedDefaultSettings(db *gorm.DB) {
defaults := []model.SystemSetting{
// Data masking group
{Key: "data_masking_enabled", Value: "true", ValueType: "boolean", Group: "data_masking", Label: "启用数据脱敏", Desc: strPtr("启用后，查询结果中的敏感字段将被替换为 *"), DefaultValue: strPtr("true"), SortOrder: 1},
{Key: "data_masking_sensitive_fields", Value: "mobile,concat_mobile,email_alias,email,password,ga_key,private_key,secret,secret_key,key_snow,telephone,username,real_name,token,two_factor_secret,content,phone", ValueType: "string", Group: "data_masking", Label: "默认敏感字段列表", Desc: strPtr("全局敏感字段，逗号分隔。匹配列名（不区分大小写）"), DefaultValue: strPtr("mobile,concat_mobile,email_alias,email,password,ga_key,private_key,secret,secret_key,key_snow,telephone,username,real_name,token,two_factor_secret,content,phone"), SortOrder: 2},
}
for _, s := range defaults {
var existing model.SystemSetting
if db.Where("`key` = ?", s.Key).First(&existing).Error != nil {
db.Create(&s)
}
}
}
func strPtr(s string) *string {
return &s
}
backend/internal/handler/ghost.go
+2 -2
View File
@@ -457,7 +457,7 @@ func ListSettings(c *gin.Context) {
// PUT /api/v1/settings — batch update settings
func UpdateSettings(c *gin.Context) {
var items []struct {
ID int `json:"id"`
Key string `json:"key"`
Value string `json:"value"`
}
if err := c.ShouldBindJSON(&items); err != nil {
@@ -465,7 +465,7 @@ func UpdateSettings(c *gin.Context) {
return
}
for _, item := range items {
model.DB.Model(&model.SystemSetting{}).Where("id = ?", item.ID).Update("value", item.Value)
model.DB.Model(&model.SystemSetting{}).Where("`key` = ?", item.Key).Update("value", item.Value)
}
c.JSON(http.StatusOK, gin.H{"message": "Settings updated"})
}
backend/internal/handler/instance_detail.go
+101
View File
@@ -550,6 +550,7 @@ func ExecuteQuery(c *gin.Context) {
}
result["cost_ms"] = costMs
rowsInterface, _ := result["rows"].([]interface{})
applyDataMasking(result, &instance)
saveQueryHistory(userID, input.InstanceID, input.Database, input.SQL, costMs, len(rowsInterface))
c.JSON(http.StatusOK, result)
@@ -565,6 +566,7 @@ func ExecuteQuery(c *gin.Context) {
if rowsArr, ok := result["rows"].([]interface{}); ok {
rowCount = len(rowsArr)
}
applyDataMasking(result, &instance)
saveQueryHistory(userID, input.InstanceID, input.Database, input.SQL, costMs, rowCount)
c.JSON(http.StatusOK, result)
@@ -577,6 +579,7 @@ func ExecuteQuery(c *gin.Context) {
}
result["cost_ms"] = costMs
rowsInterface, _ := result["rows"].([]interface{})
applyDataMasking(result, &instance)
saveQueryHistory(userID, input.InstanceID, input.Database, input.SQL, costMs, len(rowsInterface))
c.JSON(http.StatusOK, result)
@@ -1316,3 +1319,101 @@ func saveQueryHistory(userID int, instanceID int, database string, sqlStr string
}
model.DB.Create(&history)
}
// applyDataMasking masks sensitive fields in query results with "*".
// It reads global settings (data_masking_enabled + sensitive fields) and
// merges with instance-level mask_fields.
func applyDataMasking(result gin.H, instance *model.Instance) {
// Check global enable flag
var enabledSetting model.SystemSetting
if model.DB.Where("`key` = ?", "data_masking_enabled").First(&enabledSetting).Error != nil {
return // setting not found, skip masking
}
if enabledSetting.Value != "true" {
return // masking disabled
}
// Collect global sensitive fields
var fieldsSetting model.SystemSetting
globalFields := map[string]bool{}
if model.DB.Where("`key` = ?", "data_masking_sensitive_fields").First(&fieldsSetting).Error == nil {
for _, f := range strings.Split(fieldsSetting.Value, ",") {
f = strings.TrimSpace(f)
if f != "" {
globalFields[strings.ToLower(f)] = true
}
}
}
// Merge instance-level mask_fields
instanceFields := map[string]bool{}
if instance.MaskFields != nil && *instance.MaskFields != "" {
for _, f := range strings.Split(*instance.MaskFields, ",") {
f = strings.TrimSpace(f)
if f != "" {
instanceFields[strings.ToLower(f)] = true
}
}
}
// Combine all mask fields
allMaskFields := make(map[string]bool)
for k := range globalFields {
allMaskFields[k] = true
}
for k := range instanceFields {
allMaskFields[k] = true
}
if len(allMaskFields) == 0 {
return
}
// Apply masking to rows and collect masked column names
rows, ok := result["rows"].([]interface{})
if !ok {
return
}
maskedCols := []string{}
for _, rowIface := range rows {
row, ok := rowIface.(map[string]interface{})
if !ok {
continue
}
for colName, val := range row {
if allMaskFields[strings.ToLower(colName)] {
row[colName] = maskValue(val)
// Track masked column (deduplicate)
found := false
for _, mc := range maskedCols {
if mc == colName {
found = true
break
}
}
if !found {
maskedCols = append(maskedCols, colName)
}
}
}
}
if len(maskedCols) > 0 {
result["masked_columns"] = maskedCols
}
}
// maskValue replaces a value with asterisks, preserving length info for strings
func maskValue(val interface{}) string {
if val == nil {
return "***"
}
s := fmt.Sprintf("%v", val)
if len(s) == 0 {
return "*"
}
// Replace all chars with *, keep reasonable length
if len(s) > 20 {
return "********************"
}
return strings.Repeat("*", len(s))
}
frontend/src/components/EditInstanceDialog.tsx
+24 -46
View File
@@ -1,5 +1,5 @@
import { useState, useCallback, useEffect } from 'react';
import { X, Loader2, Plus, Trash2 } from 'lucide-react';
import { X, Loader2 } from 'lucide-react';
import { instanceApi } from '@/api/instance';
import { workflowApi } from '@/api/workflow';
@@ -19,7 +19,7 @@ interface InstanceData {
workflow_id?: number | null;
workflow_name?: string | null;
excluded_databases?: string[] | null;
mask_fields?: { table: string; field: string; rule: string }[] | null;
mask_fields?: string | null;
group_id?: number | null;
}
@@ -52,11 +52,11 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
const [workflowId, setWorkflowId] = useState<number | null>(instance.workflow_id || null);
const [workflows, setWorkflows] = useState<WorkflowItem[]>([]);
// excluded_databases: comma-separated input
const [excludedDbs, setExcludedDbs] = useState((instance.excluded_databases || []).join(', '));
// mask_fields
const [maskFields, setMaskFields] = useState<{ table: string; field: string; rule: string }[]>(
instance.mask_fields && instance.mask_fields.length > 0 ? instance.mask_fields : []
const [excludedDbs, setExcludedDbs] = useState(
Array.isArray(instance.excluded_databases) ? (instance.excluded_databases as string[]).join(', ') : (instance.excluded_databases || '')
);
// mask_fields: comma-separated field names
const [maskFields, setMaskFields] = useState(instance.mask_fields || '');
const [submitting, setSubmitting] = useState(false);
const [error, setError] = useState('');
@@ -71,8 +71,10 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
setEnv(instance.env || 'development');
setAccessMode(instance.access_mode || 'readwrite');
setWorkflowId(instance.workflow_id || null);
setExcludedDbs((instance.excluded_databases || []).join(', '));
setMaskFields(instance.mask_fields && instance.mask_fields.length > 0 ? instance.mask_fields : []);
setExcludedDbs(
Array.isArray(instance.excluded_databases) ? (instance.excluded_databases as string[]).join(', ') : (instance.excluded_databases || '')
);
setMaskFields(instance.mask_fields || '');
setError('');
}, [instance]);
@@ -86,14 +88,6 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
}
}, [open]);
const addMaskField = () => setMaskFields([...maskFields, { table: '', field: '', rule: 'mask' }]);
const removeMaskField = (idx: number) => setMaskFields(maskFields.filter((_, i) => i !== idx));
const updateMaskField = (idx: number, key: string, value: string) => {
const next = [...maskFields];
next[idx] = { ...next[idx], [key]: value };
setMaskFields(next);
};
const handleSubmit = useCallback(async () => {
if (!name.trim()) { setError('请填写名称'); return; }
if (!host.trim()) { setError('请填写地址'); return; }
@@ -102,9 +96,6 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
const parsedExcluded = excludedDbs.trim()
? excludedDbs.split(',').map(s => s.trim()).filter(Boolean)
: undefined;
const parsedMask = maskFields.some(m => m.table && m.field)
? maskFields.filter(m => m.table && m.field)
: undefined;
const payload: Record<string, unknown> = {
name: name.trim(),
@@ -116,7 +107,7 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
access_mode: accessMode,
workflow_id: workflowId,
excluded_databases: parsedExcluded,
mask_fields: parsedMask,
mask_fields: maskFields.trim() || undefined,
};
if (mode === 'clone') {
payload.db_type = instance.db_type;
@@ -232,33 +223,20 @@ export default function EditInstanceDialog({ instance, open, onClose, onSaved, m
<input className={inputCls} value={excludedDbs} onChange={e => setExcludedDbs(e.target.value)} placeholder="sys, information_schema, mysql" />
</div>
{/* 脱敏字段 */}
{/* 脱敏字段 - 简化为逗号分隔输入 */}
<div>
<div className="flex items-center justify-between mb-1">
<label className={labelCls}>脱敏字段</label>
<button onClick={addMaskField} className="text-xs text-blue-400 hover:text-blue-300 flex items-center gap-1">
<Plus className="w-3 h-3" /> 添加
</button>
</div>
{maskFields.length === 0 && (
<div className="text-xs text-[var(--dg-text-muted)] py-2">未配置脱敏规则</div>
)}
<div className="space-y-2">
{maskFields.map((mf, idx) => (
<div key={idx} className="flex items-center gap-2">
<input className="flex-1 px-2 py-1.5 bg-[var(--dg-bg-tertiary)] border border-[var(--dg-border)] rounded text-xs focus:border-blue-400 focus:outline-none" value={mf.table} onChange={e => updateMaskField(idx, 'table', e.target.value)} placeholder="表名" />
<input className="flex-1 px-2 py-1.5 bg-[var(--dg-bg-tertiary)] border border-[var(--dg-border)] rounded text-xs focus:border-blue-400 focus:outline-none" value={mf.field} onChange={e => updateMaskField(idx, 'field', e.target.value)} placeholder="字段名" />
<select className="w-20 px-2 py-1.5 bg-[var(--dg-bg-tertiary)] border border-[var(--dg-border)] rounded text-xs focus:border-blue-400 focus:outline-none" value={mf.rule} onChange={e => updateMaskField(idx, 'rule', e.target.value)}>
<option value="mask">遮盖</option>
<option value="hide">隐藏</option>
<option value="hash">哈希</option>
</select>
<button onClick={() => removeMaskField(idx)} className="p-1 text-[var(--dg-text-muted)] hover:text-red-400">
<Trash2 className="w-3.5 h-3.5" />
</button>
</div>
))}
</div>
<label className={labelCls}>
自定义脱敏字段 <span className="font-normal text-[var(--dg-text-muted)]">（逗号分隔，与全局敏感字段合并，如 id_card, address）</span>
</label>
<input
className={inputCls}
value={maskFields}
onChange={e => setMaskFields(e.target.value)}
placeholder="id_card, address, bank_account"
/>
<p className="text-xs text-[var(--dg-text-muted)] mt-1">
仅需添加此实例特有的脱敏字段，全局敏感字段（mobile, email 等）已在系统设置中统一配置
</p>
</div>
</div>
frontend/src/pages/PlatformConfigPage.tsx
+1
View File
@@ -82,6 +82,7 @@ function CleanupActions() {
const GROUP_META: Record<string, { label: string; icon: string; desc: string }> = {
query: { label: '查询控制', icon: '🔍', desc: 'LIMIT 上限、超时、自动 LIMIT' },
security: { label: '安全策略', icon: '🔒', desc: '登录安全、密码策略' },
data_masking: { label: '数据脱敏', icon: '🛡️', desc: '控制查询结果中的敏感字段脱敏' },
cleanup: { label: '日志清理', icon: '🧹', desc: '工单、审计日志、查询历史保留天数' },
executor: { label: '执行器', icon: '⚙️', desc: 'SQL 执行超时' },
};
frontend/src/pages/QueriesPage.tsx
+6 -14
View File
@@ -32,6 +32,7 @@ interface QueryResult {
raw_key?: string;
raw_text?: string;
error?: string;
masked_columns?: string[];
}
interface QueryHistoryItem {
@@ -64,7 +65,8 @@ const dbTypeIcon: Record<string, string> = {
mongodb: '/db-logos/mongo.svg',
};
const sensitiveKeywords = ['password', 'pwd', 'secret', 'token', 'phone', 'mobile', 'email', 'id_card', 'card_no'];
// 脱敏关键词不再硬编码——由后端 applyDataMasking 返回 masked_columns 标记
// 后端根据 data_masking_enabled + data_masking_sensitive_fields + 实例 mask_fields 决定
const placeholderMap: Record<DBType, string> = {
mysql: "-- MySQL 查询\n-- 仅支持 SELECT, SHOW, EXPLAIN, DESCRIBE\n\nSELECT id, name, status\nFROM users\nWHERE status = 'active'\nLIMIT 100;",
@@ -76,16 +78,6 @@ const placeholderMap: Record<DBType, string> = {
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
function maskData(value: string): string {
if (!value) return '***';
if (typeof value === 'string' && value.includes('@')) {
const [name, domain] = value.split('@');
return name.substring(0, 3) + '***@' + domain;
}
if (typeof value === 'string' && value.length > 7) return value.substring(0, 3) + '****' + value.substring(value.length - 4);
return '***';
}
function exportToCSV(columns: string[], rows: Record<string, unknown>[]): string {
const header = columns.join(',');
const lines = rows.map(row =>
@@ -534,7 +526,7 @@ export default function QueriesPage() {
className="text-left px-3 py-2 font-semibold text-xs cursor-pointer hover:text-blue-400 transition-colors select-none whitespace-nowrap">
<div className="flex items-center gap-1">
{col} <SortIcon col={col} />
{sensitiveKeywords.some(k => col.toLowerCase().includes(k)) && (
{result.masked_columns?.includes(col) && (
<span className="ml-1 px-1 py-0.5 bg-yellow-500/10 text-yellow-400 text-[9px] rounded">敏</span>
)}
</div>
@@ -550,12 +542,12 @@ export default function QueriesPage() {
<tr key={ri} className="dg-table-row" style={{ height: rowHeight }}>
<td className="px-3 py-2 text-[10px] text-[var(--dg-text-muted)] font-mono">{ri + 1}</td>
{result.columns.map(col => {
const isSensitive = sensitiveKeywords.some(k => col.toLowerCase().includes(k));
const isMasked = result.masked_columns?.includes(col);
const cellValue = row[col] ?? '';
const { text: displayText, isJson } = formatCellValue(cellValue);
return (
<td key={col} className={`px-3 py-2 text-xs ${isJson ? 'max-w-[500px] whitespace-pre-wrap break-all font-mono text-[11px] leading-relaxed' : 'whitespace-pre-wrap break-words'}`} title={String(cellValue)}>
{isSensitive ? <span className="text-yellow-400 italic">{maskData(String(cellValue))}</span> :
{isMasked ? <span className="text-yellow-400 italic">{String(cellValue)}</span> :
isJson ? <span className="text-blue-300">{displayText}</span> :
String(cellValue)}
</td>