diff --git a/scripts/release.sh b/scripts/release.sh index f9f01e9..5cdf35b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -25,6 +25,12 @@ # bash scripts/release.sh --pin # 额外把 deployment.yaml 钉到不可变 tag(可选) # bash scripts/release.sh --dry-run # 只检查锁与前置条件,不构建不推送 # +# 部署后自愈与版本戳: +# 滚动完成后核对 Pod digest 与远端 latest;不一致时自动删 Pod 强制重拉 +# (节点镜像缓存/拉取时机问题),仍不一致才 exit 1。 +# 核对通过后给 Deployment 打三个 annotation,直接看集群就知道线上跑哪版: +# lokistack.io/release-image / release-tag / release-time +# # 并发保护(重要): # 本项目所有环境都拉 latest,两个构建同时跑会互相覆盖 latest: # 后完成的那次生效,先完成的那次白推。曾出现"中断的构建后来完成, @@ -153,17 +159,44 @@ if [ "${MODE}" != "--no-deploy" ]; then kubectl -n log rollout restart deploy/lokistack kubectl -n log rollout status deploy/lokistack --timeout=300s - POD_DIGEST=$(kubectl -n log get po -l app=lokistack \ - -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null | sed 's/.*@//') + pod_digest() { + kubectl -n log get po -l app=lokistack \ + -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null | sed 's/.*@//' + } + + POD_DIGEST=$(pod_digest) echo "" echo "==========================================" echo " 部署完成,核对 Pod 与远端 latest:" echo " 远端 latest : ${D_LATEST:0:19}" echo " Pod 运行中 : ${POD_DIGEST:0:19}" + + # ── 自愈:digest 不一致时强制重拉一次 ── + # 常见原因:节点缓存了旧的 latest 映射、或 rollout 与 push 完成时间太接近。 + # 直接删 Pod 让 kubelet 重新解析 latest 通常就恢复了,不需要人工介入。 + if [ "${POD_DIGEST:0:19}" != "${D_LATEST:0:19}" ]; then + echo " ⚠️ 不一致,删除 Pod 强制重新拉取 latest..." + kubectl -n log delete po -l app=lokistack --wait=false >/dev/null 2>&1 || true + for _ in $(seq 1 30); do + sleep 8 + POD_DIGEST=$(pod_digest) + [ "${POD_DIGEST:0:19}" = "${D_LATEST:0:19}" ] && break + done + echo " 重拉后 Pod : ${POD_DIGEST:0:19}" + fi + if [ "${POD_DIGEST:0:19}" = "${D_LATEST:0:19}" ]; then echo " → 一致 ✅" + # ── 版本戳:下次排查"线上到底跑哪版"直接看 annotation ── + kubectl -n log annotate deploy/lokistack --overwrite \ + "lokistack.io/release-image=${D_LATEST#sha256:}" \ + "lokistack.io/release-tag=${IMMUTABLE_TAG}" \ + "lokistack.io/release-time=$(date -Is)" >/dev/null 2>&1 \ + && echo " 已标注 run 版本:${D_LATEST:0:19} (${IMMUTABLE_TAG})" else - echo " → 不一致 ❌(Pod 可能还在拉取,或节点镜像缓存异常)" + echo " → 不一致 ❌(重拉后仍不符,检查节点镜像缓存 / 网络)" + echo "==========================================" + exit 1 fi echo "==========================================" fi