From 738498fea33b378141371380bb56f4b0fe5ca957 Mon Sep 17 00:00:00 2001 From: "Logan.Li" Date: Mon, 26 Oct 2026 22:09:33 +0800 Subject: [PATCH] feat(k8s): align install.sh with latest opsx-loki version - cloud auto-detection (ACK/TKE/EKS/CCE/GKE/AKS/K3s/self-hosted) via node providerID, used for deployment banner and failure diagnostics - storage class resolution chain: explicit STORAGE_CLASS > default annotation > sole SC; self-hosted clusters without any SC now get a zero-dependency static local PV (hostPath DirectoryOrCreate + nodeAffinity pin + WaitForFirstConsumer + Retain), LOCAL_STORAGE auto/off/force tri-state - PVC rendered with resolved storageClassName + PVC_SIZE; image rendered into deployment via IMAGE env - rollout failure diagnostics per waiting-reason: ImagePullBackOff lists per-cloud registry suggestions, CreateContainerConfigError points to secret, PVC Pending suggests per-cloud SC - colored info/ok/warn/die helpers, /dev/urandom key fallback, completion banner with pod/node/ip, port-forward and backup hints - kubernetes/examples/: alibaba ALB + tencent qcloud (CLB health check annotations, NodePort backend) ingress templates, secret example moved here (opsx-loki layout) - README updated with env-var table and local-storage notes --- kubernetes/examples/ingress-tke.yaml | 32 ++ kubernetes/examples/ingress.yaml | 23 + kubernetes/{ => examples}/secret.example.yaml | 7 +- kubernetes/install.sh | 439 ++++++++++++------ 4 files changed, 367 insertions(+), 134 deletions(-) create mode 100644 kubernetes/examples/ingress-tke.yaml create mode 100644 kubernetes/examples/ingress.yaml rename kubernetes/{ => examples}/secret.example.yaml (56%) diff --git a/kubernetes/examples/ingress-tke.yaml b/kubernetes/examples/ingress-tke.yaml new file mode 100644 index 0000000..ed0ca19 --- /dev/null +++ b/kubernetes/examples/ingress-tke.yaml @@ -0,0 +1,32 @@ +# 腾讯云 TKE 专用 Ingress 模板 +# +# 与通用 ingress.yaml 的区别: +# 通用用 nginx/alb class → TKE 不支持 +# TKE 用 ingressClassName: qcloud,且后端 Service 必须是 NodePort 类型 +# +# 使用前: +# 1) 先创建 NodePort Service:kubectl apply -f kubernetes/service-nodeport.yaml +# 2) 把下面 host 改成你要用的域名 +# 3) kubectl apply -f kubernetes/examples/ingress-tke.yaml +# 4) 在腾讯云控制台/DNS 把该域名解析到 CLB 的公网 IP +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: opsxsql + namespace: opsxsql + annotations: + cloud.tencent.com/health-check-path: /health + cloud.tencent.com/health-check-interval: "10" +spec: + ingressClassName: qcloud + rules: + - host: opsxsql.example.com # ← 改成你的域名 + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opsxsql-nodeport + port: + number: 80 diff --git a/kubernetes/examples/ingress.yaml b/kubernetes/examples/ingress.yaml new file mode 100644 index 0000000..e798ed5 --- /dev/null +++ b/kubernetes/examples/ingress.yaml @@ -0,0 +1,23 @@ +# 可选:阿里云 ACK 用 ALB IngressClass 的模板 +# 其他云:腾讯 TKE 用 examples/ingress-tke.yaml;自建/nginx 用仓库根 kubernetes/ingress.yaml +# 若继续走外部 WAF 回源到 Service,则不需要本文件 +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: opsxsql + namespace: opsxsql + annotations: + alb.ingress.kubernetes.io/healthcheck-path: /health +spec: + ingressClassName: alb + rules: + - host: opsxsql.example.com # ← 改成你的域名 + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opsxsql + port: + number: 80 diff --git a/kubernetes/secret.example.yaml b/kubernetes/examples/secret.example.yaml similarity index 56% rename from kubernetes/secret.example.yaml rename to kubernetes/examples/secret.example.yaml index d5cecc9..fff9e4c 100644 --- a/kubernetes/secret.example.yaml +++ b/kubernetes/examples/secret.example.yaml @@ -1,4 +1,5 @@ -# 密钥模板 —— 建议用 kubectl create secret 命令创建(install.sh 已自动处理),不要把真实值提交到仓库 +# 只做模板:不要提交真实密钥。 +# 推荐直接用命令创建(避免密钥落盘到 Git;install.sh 已自动处理): # kubectl -n opsxsql create secret generic opsxsql-secrets \ # --from-literal=SECRET_KEY="$(openssl rand -hex 32)" \ # --from-literal=JWT_SECRET="$(openssl rand -hex 32)" @@ -9,5 +10,5 @@ metadata: namespace: opsxsql type: Opaque stringData: - SECRET_KEY: "change-me-openssl-rand-hex-32" - JWT_SECRET: "change-me-openssl-rand-hex-32" + SECRET_KEY: "REPLACE_WITH_32_BYTES_HEX" + JWT_SECRET: "REPLACE_WITH_32_BYTES_HEX" diff --git a/kubernetes/install.sh b/kubernetes/install.sh index 3cf0ae9..7016b22 100755 --- a/kubernetes/install.sh +++ b/kubernetes/install.sh @@ -1,165 +1,342 @@ -#!/bin/bash -# ============================================================ -# opsxsql 一键部署到 Kubernetes(参照 opsx-loki / deploy-loki 模式) -# 用法: -# bash kubernetes/install.sh # 默认部署(命名空间 opsxsql + ClusterIP) -# bash kubernetes/install.sh --nodeport # 附带 NodePort 30900 入口 -# bash kubernetes/install.sh --ingress opsxsql.example.com # 附带 Ingress -# IMAGE=registry.example.com/opsxsql:v1 bash kubernetes/install.sh # 指定镜像 -# 幂等: 重复执行安全;密钥已存在则复用(不会轮换,已加密数据不受影响) -# ============================================================ +#!/usr/bin/env bash +# ============================================================================ +# opsxsql 一键部署(参照 opsx-loki 最新 install.sh) +# +# 自动适配:阿里云 ACK / 腾讯云 TKE / AWS EKS / 华为云 CCE / GCP GKE / Azure AKS / K3s / 自建 +# 幂等:重复执行不覆盖密钥、不重建数据卷 +# +# 用法: +# bash kubernetes/install.sh # 自动探测,默认镜像 opsxsql:latest +# IMAGE=registry.example.com/opsxsql:v1 bash kubernetes/install.sh +# WITH_NODEPORT=1 bash kubernetes/install.sh # 同时开 NodePort(:30900) +# bash kubernetes/install.sh --nodeport # 等价于上一行 +# bash kubernetes/install.sh --ingress opsxsql.example.com # 通用 nginx Ingress +# STORAGE_CLASS=local-path NS=opsxsql PVC_SIZE=20Gi bash kubernetes/install.sh +# ============================================================================ set -euo pipefail NS="${NS:-opsxsql}" IMAGE="${IMAGE:-opsxsql:latest}" SECRET_NAME="opsxsql-secrets" -WITH_NODEPORT=0 +WITH_NODEPORT="${WITH_NODEPORT:-0}" INGRESS_HOST="" +STORAGE_CLASS="${STORAGE_CLASS:-}" +PVC_SIZE="${PVC_SIZE:-10Gi}" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# ---- 参数解析 ---- +info() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } +ok() { printf '\033[1;32m✅\33[0m %s\n' "$*"; } +warn() { printf '\033[1;33m⚠️ \33[0m %s\n' "$*"; } +die() { printf '\033[1;31m❌\33[0m %s\n' "$*" >&2; exit 1; } + +# ---- 参数解析(兼容 flags 与环境变量) ---- while [[ $# -gt 0 ]]; do case "$1" in --nodeport) WITH_NODEPORT=1; shift ;; --ingress) INGRESS_HOST="${2:?--ingress 需要域名参数}"; shift 2 ;; - -h|--help) - sed -n '2,10p' "$0"; exit 0 ;; - *) echo "未知参数: $1(看 --help)"; exit 1 ;; + -h|--help) sed -n '2,17p' "$0"; exit 0 ;; + *) die "未知参数: $1(看 --help)" ;; esac done -# ---- 前置检查 ---- -for cmd in kubectl openssl; do - command -v "$cmd" >/dev/null 2>&1 || { echo "缺少依赖: $cmd"; exit 1; } -done -kubectl cluster-info >/dev/null 2>&1 || { echo "kubectl 无法连接集群"; exit 1; } +# ============================================================================ +# 0) 前置检查 +# ============================================================================ +command -v kubectl >/dev/null 2>&1 || die "未找到 kubectl,请先安装" +kubectl version --request-timeout=5s >/dev/null 2>&1 || die "kubectl 无法连接集群,请检查 kubeconfig" -echo "==> 部署 opsxsql 到命名空间 ${NS}(镜像 ${IMAGE})" +# ============================================================================ +# 0.1) 云平台识别(信息展示 + 失败诊断用) +# ============================================================================ +detect_cloud() { + local pid="" + pid=$(kubectl get nodes -o jsonpath='{.items[0].spec.providerID}' 2>/dev/null || true) + case "${pid}" in + qcloud://*) echo "腾讯云 TKE" ;; + aws://*) echo "AWS EKS" ;; + aliyun://*) echo "阿里云 ACK" ;; + huaweicloud*|huawei*) echo "华为云 CCE" ;; + gce://*) echo "GCP GKE" ;; + azure://*) echo "Azure AKS" ;; + k3s://*) echo "K3s(自建)" ;; + *) echo "自建 / 其他 K8s" ;; + esac +} -# ---- 1) 命名空间 ---- -kubectl apply -f - < default 注解 > 唯一 SC > 失败 +# 结果写入全局 RESOLVED_SC / SC_ERROR(不要用 $() 调用,子 shell 赋值不回传) +# ============================================================================ +RESOLVED_SC="" +SC_ERROR="" +resolve_storage_class() { + local sc="" all="" count=0 + RESOLVED_SC=""; SC_ERROR="" + + if [ -n "${STORAGE_CLASS}" ]; then + if ! kubectl get sc "${STORAGE_CLASS}" >/dev/null 2>&1; then + SC_ERROR="指定的 StorageClass '${STORAGE_CLASS}' 不存在。现有:$(kubectl get sc -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' 2>/dev/null)" + return 1 + fi + RESOLVED_SC="${STORAGE_CLASS}"; return 0 + fi + + sc=$(kubectl get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -1 || true) + if [ -n "${sc}" ]; then RESOLVED_SC="${sc}"; return 0; fi + + all=$(kubectl get sc -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) + count=$(printf '%s' "${all}" | grep -c . || true) + if [ "${count}" = "1" ]; then RESOLVED_SC="${all}"; return 0; fi + + return 1 +} + +# ============================================================================ +# 0.3) 自建集群兜底:无任何 StorageClass 时创建「静态 local PV」(零外部依赖) +# hostPath + DirectoryOrCreate + nodeAffinity 锁节点 + WaitForFirstConsumer + Retain +# ============================================================================ +LOCAL_SC_NAME="opsxsql-local" +LOCAL_PV_NAME="opsxsql-data-pv" +LOCAL_PV_SIZE="${PVC_SIZE}" +LOCAL_PV_PATH="${PV_PATH:-/var/lib/opsxsql-data}" + +pick_local_node() { + if [ -n "${NODE_NAME:-}" ]; then echo "${NODE_NAME}"; return 0; fi + kubectl get nodes --no-headers \ + -o custom-columns=NAME:.metadata.name,READY:.status.conditions[-1].status 2>/dev/null \ + | awk '$2=="True"{print $1; exit}' +} + +create_local_storage() { + local node + node="$(pick_local_node)" + [ -n "${node}" ] || { warn "找不到 Ready 节点,无法创建本地存储"; return 1; } + + info "创建节点本地存储:节点=${node} 路径=${LOCAL_PV_PATH} 容量=${LOCAL_PV_SIZE}" + + cat </dev/null +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${LOCAL_SC_NAME} +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Retain +EOF + + cat </dev/null +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${LOCAL_PV_NAME} + labels: + app: opsxsql +spec: + capacity: + storage: ${LOCAL_PV_SIZE} + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: ${LOCAL_SC_NAME} + hostPath: + path: ${LOCAL_PV_PATH} + type: DirectoryOrCreate + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - ${node} +EOF + + ok "已创建 StorageClass ${LOCAL_SC_NAME} + PV ${LOCAL_PV_NAME}(节点 ${node})" + warn "注意:数据固定在该节点 ${LOCAL_PV_PATH},Pod 不会漂移到其他节点" + RESOLVED_SC="${LOCAL_SC_NAME}" + return 0 +} + +CLOUD="$(detect_cloud)" +info "opsxsql 部署开始(namespace: ${NS},镜像 ${IMAGE})" +info "集群: $(kubectl config current-context 2>/dev/null || echo unknown)" +info "平台: ${CLOUD}" + +# LOCAL_STORAGE 三态:auto(默认,无 SC 时自动建) / off(只提示) / force(强制建) +LOCAL_STORAGE="${LOCAL_STORAGE:-auto}" + +if [ "${LOCAL_STORAGE}" = "force" ]; then + warn "LOCAL_STORAGE=force:强制使用节点本地存储" + create_local_storage || die "创建节点本地存储失败" +elif ! resolve_storage_class; then + if [ -n "${SC_ERROR}" ]; then + die "${SC_ERROR}" + fi + echo "" + if [ "${LOCAL_STORAGE}" = "auto" ]; then + warn "集群没有可用 StorageClass(自建集群常见)→ 自动启用节点本地存储" + create_local_storage || die "创建节点本地存储失败" + else + warn "未找到可用的 StorageClass,且 LOCAL_STORAGE=off" + echo "" + echo " 现有 StorageClass:" + kubectl get sc 2>/dev/null | sed 's/^/ /' || echo " (无)" + echo "" + echo " 解决办法(选一个):" + echo " 1) 手动指定(推荐,若已有可用 SC):" + echo " STORAGE_CLASS= bash kubernetes/install.sh" + echo " 2) 标记某个 SC 为默认:" + echo " kubectl patch sc -p '{\"metadata\":{\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'" + echo " 3) 自建集群安装 local-path-provisioner:" + echo " kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml" + echo " STORAGE_CLASS=local-path bash kubernetes/install.sh" + echo "" + die "StorageClass 未就绪,部署中止(避免 PVC 永久 Pending)" + fi +else + ok "StorageClass: ${RESOLVED_SC}" +fi + +# ============================================================================ +# 1) 命名空间 +# ============================================================================ +cat </dev/null apiVersion: v1 kind: Namespace metadata: name: ${NS} EOF -# ---- 2) 密钥(已存在则复用) ---- -if kubectl -n "$NS" get secret "$SECRET_NAME" >/dev/null 2>&1; then - echo "==> 密钥 ${SECRET_NAME} 已存在,复用(不轮换)" +# ============================================================================ +# 2) Secret(幂等:已存在则跳过,绝不覆盖 —— SECRET_KEY 换了已加密数据不可解) +# ============================================================================ +rand_hex() { + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 32 + elif [ -c /dev/urandom ]; then + head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' + else + die "无法生成随机密钥(缺 openssl 且无 /dev/urandom)" + fi +} + +if kubectl -n "${NS}" get secret "${SECRET_NAME}" >/dev/null 2>&1; then + info "Secret ${SECRET_NAME} 已存在,跳过(不覆盖现有密钥)" else - echo "==> 生成密钥(仅此一次;丢失会导致已加密的实例/LDAP 密码无法解密,请自行备份)" - kubectl -n "$NS" create secret generic "$SECRET_NAME" \ - --from-literal=SECRET_KEY="$(openssl rand -hex 32)" \ - --from-literal=JWT_SECRET="$(openssl rand -hex 32)" + info "生成随机密钥 SECRET_KEY / JWT_SECRET(丢失则已加密的实例/LDAP 密码无法解密,请自行备份)" + kubectl -n "${NS}" create secret generic "${SECRET_NAME}" \ + --from-literal=SECRET_KEY="$(rand_hex)" \ + --from-literal=JWT_SECRET="$(rand_hex)" >/dev/null + ok "Secret 已创建" fi -# ---- 3) 数据卷 + 工作负载 + Service ---- -kubectl -n "$NS" apply -f - < "${RENDER_DIR}/pvc.yaml" +kubectl apply -f "${RENDER_DIR}/pvc.yaml" -kubectl -n "$NS" apply -f - < "${RENDER_DIR}/deployment.yaml" +kubectl apply -f "${RENDER_DIR}/deployment.yaml" +kubectl apply -f "${DIR}/service.yaml" -# ---- 4) 可选入口 ---- -if [[ "$WITH_NODEPORT" == "1" ]]; then - kubectl -n "$NS" apply -f "$(dirname "$0")/service-nodeport.yaml" - echo "==> NodePort 已启用" +if [ "${WITH_NODEPORT}" = "1" ]; then + kubectl apply -f "${DIR}/service-nodeport.yaml" + ok "NodePort Service 已创建(:30900)" fi -if [[ -n "$INGRESS_HOST" ]]; then - kubectl -n "$NS" apply -f - < Ingress 已创建(${INGRESS_HOST})" +if [ -n "${INGRESS_HOST}" ]; then + sed "s|host: opsxsql.example.com.*|host: ${INGRESS_HOST}|" "${DIR}/ingress.yaml" > "${RENDER_DIR}/ingress.yaml" + kubectl apply -f "${RENDER_DIR}/ingress.yaml" + ok "Ingress 已创建(${INGRESS_HOST},nginx class;其他云参照 ${DIR}/examples/)" fi -# ---- 5) 等待就绪 ---- -echo "==> 等待 Pod 就绪..." -if kubectl -n "$NS" rollout status deployment/opsxsql --timeout=180s; then +# ============================================================================ +# 5) 等待就绪(失败时按云诊断) +# ============================================================================ +info "等待 Deployment 就绪(最多 180 秒)..." +if ! kubectl -n "${NS}" rollout status deployment/opsxsql --timeout=180s; then echo "" - echo "==========================================" - echo " opsxsql 部署完成!(命名空间 ${NS})" - echo "==========================================" - echo " 默认账号: admin / admin123(首次登录后请修改)" - echo " 访问方式:" - [[ "$WITH_NODEPORT" == "1" ]] && echo " NodePort: http://<任意节点IP>:30900" - [[ -n "$INGRESS_HOST" ]] && echo " Ingress: http://${INGRESS_HOST}" - echo " 集群内: http://opsxsql.${NS}.svc" - echo " 本地试连: kubectl -n ${NS} port-forward svc/opsxsql 9000:80" - echo " 数据备份: kubectl -n ${NS} exec deploy/opsxsql -- sqlite3 /app/data/sqlite3.db '.backup /app/data/backup.db'" - echo "==========================================" -else - echo "部署超时,排查: kubectl -n ${NS} describe pod -l app.kubernetes.io/name=opsxsql" - exit 1 + warn "部署未在 180 秒内就绪,开始诊断:" + echo "" + kubectl -n "${NS}" get po -l app.kubernetes.io/name=opsxsql -o wide 2>/dev/null | sed 's/^/ /' + + REASON=$(kubectl -n "${NS}" get po -l app.kubernetes.io/name=opsxsql -o jsonpath='{.items[0].status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true) + echo "" + case "${REASON}" in + ImagePullBackOff|ErrImagePull) + warn "镜像拉取失败(${REASON})" + echo " → 当前镜像: ${IMAGE}" + echo " → 该节点拉不到该仓库。改为目标云可拉的仓库后重跑:" + echo " 腾讯云 TCR : ccr.ccs.tencentyun.com//opsxsql:latest" + echo " 阿里云 ACR : registry.cn-hangzhou.aliyuncs.com//opsxsql:latest" + echo " 华为云 SWR : swr.cn-north-4.myhuaweicloud.com//opsxsql:latest" + echo " 自建 Harbor: harbor.your.domain/library/opsxsql:latest" + echo " 示例: IMAGE=registry.cn-hangzhou.aliyuncs.com//opsxsql:latest bash kubernetes/install.sh" + ;; + CreateContainerConfigError) + warn "容器配置缺失(通常是 Secret 不存在)" + echo " → 检查: kubectl -n ${NS} get secret ${SECRET_NAME}" + ;; + "") + PVC_PHASE=$(kubectl -n "${NS}" get pvc opsxsql-data -o jsonpath='{.status.phase}' 2>/dev/null || echo unknown) + if [ "${PVC_PHASE}" != "Bound" ]; then + warn "PVC 未绑定(phase=${PVC_PHASE})" + echo " → 检查 StorageClass: kubectl -n ${NS} describe pvc opsxsql-data" + echo " → ${CLOUD} 建议的 SC:" + case "${CLOUD}" in + 腾讯云*) echo " cbs" ;; + AWS*) echo " gp3 / gp2" ;; + 阿里云*) echo " alicloud-disk-essd" ;; + 华为云*) echo " csi-disk" ;; + *) echo " local-path(需装 local-path-provisioner)" ;; + esac + fi + ;; + esac + + echo "" + kubectl -n "${NS}" describe po -l app.kubernetes.io/name=opsxsql 2>/dev/null | tail -15 | sed 's/^/ /' + die "部署未完成,请按上面提示处理" fi +ok "Deployment 已就绪" + +# ============================================================================ +# 6) 输出访问信息 +# ============================================================================ +POD=$(kubectl -n "${NS}" get po -l app.kubernetes.io/name=opsxsql -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "-") +POD_IP=$(kubectl -n "${NS}" get po -l app.kubernetes.io/name=opsxsql -o jsonpath='{.items[0].status.podIP}' 2>/dev/null || echo "-") +NODE=$(kubectl -n "${NS}" get po -l app.kubernetes.io/name=opsxsql -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null || echo "-") + +echo "" +echo "════════════════════════════════════════════════════════" +ok "opsxsql 部署完成" +echo "════════════════════════════════════════════════════════" +echo " 平台 : ${CLOUD}" +echo " Namespace : ${NS}" +echo " 镜像 : ${IMAGE}" +echo " 存储类 : ${RESOLVED_SC}" +echo " Pod : ${POD} (node=${NODE} ip=${POD_IP})" +echo " Service : opsxsql.${NS}.svc.cluster.local:80" +[ "${WITH_NODEPORT}" = "1" ] && echo " NodePort : <任意节点IP>:30900" +[ -n "${INGRESS_HOST}" ] && echo " Ingress : http://${INGRESS_HOST}" +echo "" +echo " 初始账号 : admin / admin123 ← 首次登录请立即改密" +echo "" +echo " 本地访问 : kubectl -n ${NS} port-forward svc/opsxsql 9000:80" +echo " 浏览器打开 http://localhost:9000" +echo "" +echo " 数据备份 : kubectl -n ${NS} exec deploy/opsxsql -- sqlite3 /app/data/sqlite3.db '.backup /app/data/backup.db'" +echo " 配 Ingress(可选):参照 ${DIR}/examples/ 按云选模板" +echo "════════════════════════════════════════════════════════"