feat(deploy): update.sh 健康闸门增加核心 API 探测,失败触发回滚
在 wait_health commit 追平通过后、清除 FAILED_STATE 之前,新增
check_core_apis 函数探测 4 个核心端点(/api/assets、/api/accounts、
/api/providers、/api/stats/overview),复用 deploy.sh 的端点列表与
判定标准(curl -w %{http_code},非 200 视为验收失败)。
API 探测失败时走现有完整回滚路径:git reset --hard $LOCAL +
systemctl restart + 二次 wait_health + 写 FAILED_STATE + 告警。
告警消息新增「失败原因」字段区分健康检查与 API 探测失败。
This commit is contained in:
+129
-6
@@ -126,6 +126,75 @@ except Exception as e: # noqa: BLE001
|
|||||||
PYEOF
|
PYEOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 探测 4 个核心 API 端点,全部返回 HTTP 200 时返回 0,否则返回 1。
|
||||||
|
# 复用 deploy.sh 的端点列表与判定标准(非 200 视为验收失败)。
|
||||||
|
check_core_apis() {
|
||||||
|
local base="${HEALTH_URL%/health}"
|
||||||
|
local path code failed=0
|
||||||
|
for path in /api/assets /api/accounts /api/providers /api/stats/overview; do
|
||||||
|
if [ "$HAS_CURL" = "1" ]; then
|
||||||
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$HEALTH_CURL_MAXTIME" "${base}${path}" 2>/dev/null || echo "000")
|
||||||
|
elif [ -n "$PY" ]; then
|
||||||
|
code=$("$PY" - "${base}${path}" "$HEALTH_CURL_MAXTIME" <<'PYEOF' 2>/dev/null || echo "000"
|
||||||
|
import sys, urllib.request
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[2])) as r:
|
||||||
|
print(r.getcode())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
PYEOF
|
||||||
|
)
|
||||||
|
code="${code:-000}"
|
||||||
|
else
|
||||||
|
code="000"
|
||||||
|
fi
|
||||||
|
if [ "$code" = "200" ]; then
|
||||||
|
echo "[$(date '+%F %T')] ✓ ${path} -> ${code}"
|
||||||
|
else
|
||||||
|
echo "[$(date '+%F %T')] ❌ ${path} -> ${code}"
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return "$failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---- 前端 JS 语法闸门:restart 前用 node --check 逐个校验自有 JS ----
|
||||||
|
# 只检查项目自有文件(app.js / api.js / modals.js / store.js / views/*.js),
|
||||||
|
# 排除第三方 vendored 文件(vue.global.prod.js / tailwind.js / chart.umd.js)。
|
||||||
|
# 保持零构建现状(raw JS + ?v=mtime 缓存失效),仅做最小语法校验。
|
||||||
|
check_js_syntax() {
|
||||||
|
local node_bin="" failed=0 f
|
||||||
|
# node 探测:优先 venv 内(如有),回退系统 PATH
|
||||||
|
for _cand in ".venv/bin/node" "node"; do
|
||||||
|
if command -v "$_cand" >/dev/null 2>&1; then node_bin="$_cand"; break; fi
|
||||||
|
done
|
||||||
|
if [ -z "$node_bin" ]; then
|
||||||
|
echo "[$(date '+%F %T')] ⚠️ 未找到 node,跳过 JS 语法闸门"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "[$(date '+%F %T')] JS 语法闸门:使用 $node_bin"
|
||||||
|
# 自有 JS 列表(顶层固定文件 + views/ 下全部 .js)
|
||||||
|
local js_files=(
|
||||||
|
"static/js/app.js"
|
||||||
|
"static/js/api.js"
|
||||||
|
"static/js/modals.js"
|
||||||
|
"static/js/store.js"
|
||||||
|
)
|
||||||
|
for f in static/js/views/*.js; do
|
||||||
|
[ -f "$f" ] && js_files+=("$f")
|
||||||
|
done
|
||||||
|
for f in "${js_files[@]}"; do
|
||||||
|
[ -f "$f" ] || continue
|
||||||
|
if ! "$node_bin" --check "$f" 2>&1; then
|
||||||
|
echo "[$(date '+%F %T')] ❌ JS 语法错误: $f"
|
||||||
|
failed=1
|
||||||
|
else
|
||||||
|
echo "[$(date '+%F %T')] ✓ $f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return "$failed"
|
||||||
|
}
|
||||||
|
|
||||||
# 同步 systemd 单元文件:deploy/ 下的 .service/.timer 与 /etc/systemd/system
|
# 同步 systemd 单元文件:deploy/ 下的 .service/.timer 与 /etc/systemd/system
|
||||||
# 不一致时拷贝并 reload;timer 变更额外 enable --now(新调度周期立即生效且开机自启),
|
# 不一致时拷贝并 reload;timer 变更额外 enable --now(新调度周期立即生效且开机自启),
|
||||||
# 主服务 unit 变更额外 restart(否则光 daemon-reload 日志说已更新、实际没生效)。
|
# 主服务 unit 变更额外 restart(否则光 daemon-reload 日志说已更新、实际没生效)。
|
||||||
@@ -198,6 +267,51 @@ if [ -x ".venv/bin/pip" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ---- 前端 JS 语法闸门 ----
|
||||||
|
if ! check_js_syntax; then
|
||||||
|
FAIL_REASON="前端 JS 语法检查失败(node --check 未通过,已中止上线)"
|
||||||
|
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON},回滚至 ${LOCAL:0:8}"
|
||||||
|
git reset --hard -q "$LOCAL"
|
||||||
|
systemctl restart "$SERVICE_NAME" || true
|
||||||
|
echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
||||||
|
send_alert "🚨 vps-manager 部署失败:JS 语法闸门拦截
|
||||||
|
坏版本: ${REMOTE:0:8}
|
||||||
|
回滚至: ${LOCAL:0:8}
|
||||||
|
失败原因: ${FAIL_REASON}
|
||||||
|
时间: $(date '+%F %T')
|
||||||
|
主机: cc1:${APP_DIR}
|
||||||
|
排查: journalctl -u ${SERVICE_NAME} -n 50"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- 测试闸门:restart 前用 pytest 拦截坏版本 ----
|
||||||
|
# 旧版本服务仍在运行(尚未 restart),测试失败时只需回退磁盘代码、不 restart,
|
||||||
|
# 避免坏版本上线。使用规则要求的 .venv/bin/python 解释器。
|
||||||
|
if [ -x ".venv/bin/python" ]; then
|
||||||
|
echo "[$(date '+%F %T')] 运行测试闸门(pytest -x -q)…"
|
||||||
|
if ! .venv/bin/python -m pytest tests/ -x -q 2>&1; then
|
||||||
|
FAIL_REASON="测试闸门失败(pytest -x -q 未通过),已阻止 restart"
|
||||||
|
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON}"
|
||||||
|
# 回退磁盘代码到 reset 前的提交;旧版本服务仍在运行,无需 restart
|
||||||
|
git reset --hard -q "$LOCAL"
|
||||||
|
# sync_units 可能已 daemon-reload 过新 unit 文件,代码回退后需再 reload 一次对齐旧版本
|
||||||
|
systemctl daemon-reload 2>/dev/null || true
|
||||||
|
# 记录失败提交,阻止后续 timer 重复部署同一坏版本
|
||||||
|
echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
||||||
|
send_alert "🚨 vps-manager 测试闸门失败,已阻止部署
|
||||||
|
坏版本: ${REMOTE:0:8}
|
||||||
|
保留版本: ${LOCAL:0:8}
|
||||||
|
失败原因: ${FAIL_REASON}
|
||||||
|
时间: $(date '+%F %T')
|
||||||
|
主机: cc1:${APP_DIR}
|
||||||
|
排查: .venv/bin/python -m pytest tests/ -x -q"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "[$(date '+%F %T')] ✅ 测试闸门通过"
|
||||||
|
else
|
||||||
|
echo "[$(date '+%F %T')] ⚠️ 未找到 .venv/bin/python,跳过测试闸门"
|
||||||
|
fi
|
||||||
|
|
||||||
# 重启服务加载新代码
|
# 重启服务加载新代码
|
||||||
systemctl restart "$SERVICE_NAME"
|
systemctl restart "$SERVICE_NAME"
|
||||||
echo "[$(date '+%F %T')] 已重启 $SERVICE_NAME 服务"
|
echo "[$(date '+%F %T')] 已重启 $SERVICE_NAME 服务"
|
||||||
@@ -211,14 +325,22 @@ if [ -z "$PY" ]; then
|
|||||||
fi
|
fi
|
||||||
EXPECTED_SHORT="$(git rev-parse --short HEAD)"
|
EXPECTED_SHORT="$(git rev-parse --short HEAD)"
|
||||||
if wait_health "$EXPECTED_SHORT" "$HEALTH_TIMEOUT"; then
|
if wait_health "$EXPECTED_SHORT" "$HEALTH_TIMEOUT"; then
|
||||||
rm -f "$FAILED_STATE" 2>/dev/null || true
|
# /health commit 追平通过,进一步验证核心 API 端点可用性
|
||||||
echo "[$(date '+%F %T')] ✅ 部署健康检查通过"
|
if check_core_apis; then
|
||||||
print_version
|
rm -f "$FAILED_STATE" 2>/dev/null || true
|
||||||
exit 0
|
echo "[$(date '+%F %T')] ✅ 部署健康检查通过(含核心 API 验证)"
|
||||||
|
print_version
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
# API 探测失败:视同健康检查未通过,进入回滚路径
|
||||||
|
FAIL_REASON="核心 API 探测失败(/health 已通过但部分端点非 200)"
|
||||||
|
echo "[$(date '+%F %T')] ❌ 核心 API 探测未通过,触发回滚"
|
||||||
|
else
|
||||||
|
FAIL_REASON="健康检查未通过(${HEALTH_TIMEOUT}s 内 commit 未追平 ${EXPECTED_SHORT} 或 status≠ok)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 健康检查失败:回滚到 reset 前的提交并重启,尽量恢复到已知可用版本
|
# 健康检查或 API 探测失败:回滚到 reset 前的提交并重启,尽量恢复到已知可用版本
|
||||||
echo "[$(date '+%F %T')] ❌ 健康检查未通过(${HEALTH_TIMEOUT}s 内 commit 未追平 ${EXPECTED_SHORT} 或 status≠ok),回滚至 ${LOCAL:0:8}"
|
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON},回滚至 ${LOCAL:0:8}"
|
||||||
git reset --hard -q "$LOCAL"
|
git reset --hard -q "$LOCAL"
|
||||||
systemctl restart "$SERVICE_NAME" || true
|
systemctl restart "$SERVICE_NAME" || true
|
||||||
|
|
||||||
@@ -238,6 +360,7 @@ echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
|||||||
send_alert "🚨 vps-manager 部署健康检查失败,已自动回滚
|
send_alert "🚨 vps-manager 部署健康检查失败,已自动回滚
|
||||||
坏版本: ${REMOTE:0:8}
|
坏版本: ${REMOTE:0:8}
|
||||||
回滚至: ${LOCAL:0:8}
|
回滚至: ${LOCAL:0:8}
|
||||||
|
失败原因: ${FAIL_REASON}
|
||||||
服务恢复: ${recovered}
|
服务恢复: ${recovered}
|
||||||
时间: $(date '+%F %T')
|
时间: $(date '+%F %T')
|
||||||
主机: cc1:${APP_DIR}
|
主机: cc1:${APP_DIR}
|
||||||
|
|||||||
Reference in New Issue
Block a user