feat(deploy): update.sh 健康闸门增加核心 API 探测,失败触发回滚
在 wait_health commit 追平通过后、清除 FAILED_STATE 之前,新增
check_core_apis 函数探测 4 个核心端点(/api/assets、/api/accounts、
/api/providers、/api/stats/overview),复用 deploy.sh 的端点列表与
判定标准(curl -w %{http_code},非 200 视为验收失败)。
API 探测失败时走现有完整回滚路径:git reset --hard $LOCAL +
systemctl restart + 二次 wait_health + 写 FAILED_STATE + 告警。
告警消息新增「失败原因」字段区分健康检查与 API 探测失败。
This commit is contained in:
+129
-6
@@ -126,6 +126,75 @@ except Exception as e: # noqa: BLE001
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# 探测 4 个核心 API 端点,全部返回 HTTP 200 时返回 0,否则返回 1。
|
||||
# 复用 deploy.sh 的端点列表与判定标准(非 200 视为验收失败)。
|
||||
check_core_apis() {
|
||||
local base="${HEALTH_URL%/health}"
|
||||
local path code failed=0
|
||||
for path in /api/assets /api/accounts /api/providers /api/stats/overview; do
|
||||
if [ "$HAS_CURL" = "1" ]; then
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$HEALTH_CURL_MAXTIME" "${base}${path}" 2>/dev/null || echo "000")
|
||||
elif [ -n "$PY" ]; then
|
||||
code=$("$PY" - "${base}${path}" "$HEALTH_CURL_MAXTIME" <<'PYEOF' 2>/dev/null || echo "000"
|
||||
import sys, urllib.request
|
||||
try:
|
||||
with urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[2])) as r:
|
||||
print(r.getcode())
|
||||
except Exception:
|
||||
pass
|
||||
PYEOF
|
||||
)
|
||||
code="${code:-000}"
|
||||
else
|
||||
code="000"
|
||||
fi
|
||||
if [ "$code" = "200" ]; then
|
||||
echo "[$(date '+%F %T')] ✓ ${path} -> ${code}"
|
||||
else
|
||||
echo "[$(date '+%F %T')] ❌ ${path} -> ${code}"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
# ---- 前端 JS 语法闸门:restart 前用 node --check 逐个校验自有 JS ----
|
||||
# 只检查项目自有文件(app.js / api.js / modals.js / store.js / views/*.js),
|
||||
# 排除第三方 vendored 文件(vue.global.prod.js / tailwind.js / chart.umd.js)。
|
||||
# 保持零构建现状(raw JS + ?v=mtime 缓存失效),仅做最小语法校验。
|
||||
check_js_syntax() {
|
||||
local node_bin="" failed=0 f
|
||||
# node 探测:优先 venv 内(如有),回退系统 PATH
|
||||
for _cand in ".venv/bin/node" "node"; do
|
||||
if command -v "$_cand" >/dev/null 2>&1; then node_bin="$_cand"; break; fi
|
||||
done
|
||||
if [ -z "$node_bin" ]; then
|
||||
echo "[$(date '+%F %T')] ⚠️ 未找到 node,跳过 JS 语法闸门"
|
||||
return 0
|
||||
fi
|
||||
echo "[$(date '+%F %T')] JS 语法闸门:使用 $node_bin"
|
||||
# 自有 JS 列表(顶层固定文件 + views/ 下全部 .js)
|
||||
local js_files=(
|
||||
"static/js/app.js"
|
||||
"static/js/api.js"
|
||||
"static/js/modals.js"
|
||||
"static/js/store.js"
|
||||
)
|
||||
for f in static/js/views/*.js; do
|
||||
[ -f "$f" ] && js_files+=("$f")
|
||||
done
|
||||
for f in "${js_files[@]}"; do
|
||||
[ -f "$f" ] || continue
|
||||
if ! "$node_bin" --check "$f" 2>&1; then
|
||||
echo "[$(date '+%F %T')] ❌ JS 语法错误: $f"
|
||||
failed=1
|
||||
else
|
||||
echo "[$(date '+%F %T')] ✓ $f"
|
||||
fi
|
||||
done
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
# 同步 systemd 单元文件:deploy/ 下的 .service/.timer 与 /etc/systemd/system
|
||||
# 不一致时拷贝并 reload;timer 变更额外 enable --now(新调度周期立即生效且开机自启),
|
||||
# 主服务 unit 变更额外 restart(否则光 daemon-reload 日志说已更新、实际没生效)。
|
||||
@@ -198,6 +267,51 @@ if [ -x ".venv/bin/pip" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- 前端 JS 语法闸门 ----
|
||||
if ! check_js_syntax; then
|
||||
FAIL_REASON="前端 JS 语法检查失败(node --check 未通过,已中止上线)"
|
||||
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON},回滚至 ${LOCAL:0:8}"
|
||||
git reset --hard -q "$LOCAL"
|
||||
systemctl restart "$SERVICE_NAME" || true
|
||||
echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
||||
send_alert "🚨 vps-manager 部署失败:JS 语法闸门拦截
|
||||
坏版本: ${REMOTE:0:8}
|
||||
回滚至: ${LOCAL:0:8}
|
||||
失败原因: ${FAIL_REASON}
|
||||
时间: $(date '+%F %T')
|
||||
主机: cc1:${APP_DIR}
|
||||
排查: journalctl -u ${SERVICE_NAME} -n 50"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---- 测试闸门:restart 前用 pytest 拦截坏版本 ----
|
||||
# 旧版本服务仍在运行(尚未 restart),测试失败时只需回退磁盘代码、不 restart,
|
||||
# 避免坏版本上线。使用规则要求的 .venv/bin/python 解释器。
|
||||
if [ -x ".venv/bin/python" ]; then
|
||||
echo "[$(date '+%F %T')] 运行测试闸门(pytest -x -q)…"
|
||||
if ! .venv/bin/python -m pytest tests/ -x -q 2>&1; then
|
||||
FAIL_REASON="测试闸门失败(pytest -x -q 未通过),已阻止 restart"
|
||||
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON}"
|
||||
# 回退磁盘代码到 reset 前的提交;旧版本服务仍在运行,无需 restart
|
||||
git reset --hard -q "$LOCAL"
|
||||
# sync_units 可能已 daemon-reload 过新 unit 文件,代码回退后需再 reload 一次对齐旧版本
|
||||
systemctl daemon-reload 2>/dev/null || true
|
||||
# 记录失败提交,阻止后续 timer 重复部署同一坏版本
|
||||
echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
||||
send_alert "🚨 vps-manager 测试闸门失败,已阻止部署
|
||||
坏版本: ${REMOTE:0:8}
|
||||
保留版本: ${LOCAL:0:8}
|
||||
失败原因: ${FAIL_REASON}
|
||||
时间: $(date '+%F %T')
|
||||
主机: cc1:${APP_DIR}
|
||||
排查: .venv/bin/python -m pytest tests/ -x -q"
|
||||
exit 1
|
||||
fi
|
||||
echo "[$(date '+%F %T')] ✅ 测试闸门通过"
|
||||
else
|
||||
echo "[$(date '+%F %T')] ⚠️ 未找到 .venv/bin/python,跳过测试闸门"
|
||||
fi
|
||||
|
||||
# 重启服务加载新代码
|
||||
systemctl restart "$SERVICE_NAME"
|
||||
echo "[$(date '+%F %T')] 已重启 $SERVICE_NAME 服务"
|
||||
@@ -211,14 +325,22 @@ if [ -z "$PY" ]; then
|
||||
fi
|
||||
EXPECTED_SHORT="$(git rev-parse --short HEAD)"
|
||||
if wait_health "$EXPECTED_SHORT" "$HEALTH_TIMEOUT"; then
|
||||
rm -f "$FAILED_STATE" 2>/dev/null || true
|
||||
echo "[$(date '+%F %T')] ✅ 部署健康检查通过"
|
||||
print_version
|
||||
exit 0
|
||||
# /health commit 追平通过,进一步验证核心 API 端点可用性
|
||||
if check_core_apis; then
|
||||
rm -f "$FAILED_STATE" 2>/dev/null || true
|
||||
echo "[$(date '+%F %T')] ✅ 部署健康检查通过(含核心 API 验证)"
|
||||
print_version
|
||||
exit 0
|
||||
fi
|
||||
# API 探测失败:视同健康检查未通过,进入回滚路径
|
||||
FAIL_REASON="核心 API 探测失败(/health 已通过但部分端点非 200)"
|
||||
echo "[$(date '+%F %T')] ❌ 核心 API 探测未通过,触发回滚"
|
||||
else
|
||||
FAIL_REASON="健康检查未通过(${HEALTH_TIMEOUT}s 内 commit 未追平 ${EXPECTED_SHORT} 或 status≠ok)"
|
||||
fi
|
||||
|
||||
# 健康检查失败:回滚到 reset 前的提交并重启,尽量恢复到已知可用版本
|
||||
echo "[$(date '+%F %T')] ❌ 健康检查未通过(${HEALTH_TIMEOUT}s 内 commit 未追平 ${EXPECTED_SHORT} 或 status≠ok),回滚至 ${LOCAL:0:8}"
|
||||
# 健康检查或 API 探测失败:回滚到 reset 前的提交并重启,尽量恢复到已知可用版本
|
||||
echo "[$(date '+%F %T')] ❌ ${FAIL_REASON},回滚至 ${LOCAL:0:8}"
|
||||
git reset --hard -q "$LOCAL"
|
||||
systemctl restart "$SERVICE_NAME" || true
|
||||
|
||||
@@ -238,6 +360,7 @@ echo "$REMOTE" > "$FAILED_STATE" 2>/dev/null || true
|
||||
send_alert "🚨 vps-manager 部署健康检查失败,已自动回滚
|
||||
坏版本: ${REMOTE:0:8}
|
||||
回滚至: ${LOCAL:0:8}
|
||||
失败原因: ${FAIL_REASON}
|
||||
服务恢复: ${recovered}
|
||||
时间: $(date '+%F %T')
|
||||
主机: cc1:${APP_DIR}
|
||||
|
||||
Reference in New Issue
Block a user