feat: 综合平台services多服务支持+SW网络优先缓存修复+适配层/定时任务完善
This commit is contained in:
+146
-61
@@ -3,7 +3,7 @@
|
||||
统一处理 Asset 主表与其一对一详情表(VPSDetail/DomainDetail/AIAccount)的联动。
|
||||
"""
|
||||
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
@@ -30,6 +30,10 @@ from app.schemas.asset import (
|
||||
VPSDetailRead,
|
||||
)
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("vps-manager.assets")
|
||||
|
||||
# 资产类型 -> (AssetCreate/Update 中的字段名, 详情表模型)
|
||||
DETAIL_MAP = {
|
||||
AssetType.VPS: ("vps_detail", VPSDetail),
|
||||
@@ -140,11 +144,49 @@ def _apply_detail_update(asset_type: AssetType, existing, detail_in) -> None:
|
||||
setattr(existing, key, value)
|
||||
|
||||
|
||||
def _get_details_batch(session: Session, assets: list) -> dict:
|
||||
"""批量预加载所有资产的详情和 Provider 名,避免 N+1 查询"""
|
||||
if not assets:
|
||||
return {}
|
||||
asset_ids = [a.id for a in assets]
|
||||
# 批量查 Provider 名
|
||||
provider_ids = {a.provider_id for a in assets if a.provider_id}
|
||||
provider_map = {}
|
||||
if provider_ids:
|
||||
for p in session.exec(select(Provider).where(Provider.id.in_(provider_ids))).all():
|
||||
provider_map[p.id] = p.name
|
||||
# 批量查各类型详情
|
||||
detail_map = {}
|
||||
for asset_type, (_, model) in DETAIL_MAP.items():
|
||||
typed_ids = [a.id for a in assets if a.asset_type == asset_type]
|
||||
if typed_ids:
|
||||
for d in session.exec(select(model).where(model.asset_id.in_(typed_ids))).all():
|
||||
detail_map[d.asset_id] = d
|
||||
return {"providers": provider_map, "details": detail_map}
|
||||
|
||||
|
||||
def _to_read_batch(session: Session, asset: Asset, batch: dict) -> AssetRead:
|
||||
"""用批量预加载的数据组装 AssetRead(避免逐个查询)"""
|
||||
read = AssetRead.model_validate(asset)
|
||||
read.days_to_expiry = _days_to_expiry(asset.expiry_date)
|
||||
read.provider_name = batch["providers"].get(asset.provider_id)
|
||||
detail = batch["details"].get(asset.id)
|
||||
if isinstance(detail, VPSDetail):
|
||||
read.vps_detail = _detail_to_read(detail)
|
||||
elif isinstance(detail, DomainDetail):
|
||||
read.domain_detail = _detail_to_read(detail)
|
||||
elif isinstance(detail, AIAccount):
|
||||
read.ai_detail = _detail_to_read(detail)
|
||||
elif isinstance(detail, CloudflareDetail):
|
||||
read.cloudflare_detail = _detail_to_read(detail)
|
||||
return read
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CRUD
|
||||
# --------------------------------------------------------------------------- #
|
||||
def create_asset(session: Session, data: AssetCreate) -> AssetRead:
|
||||
"""创建资产及其详情"""
|
||||
def _create_asset_no_commit(session: Session, data: AssetCreate) -> Asset:
|
||||
"""创建资产及详情(不 commit,由调用方统一提交)"""
|
||||
item = DETAIL_MAP.get(data.asset_type)
|
||||
detail_in = None
|
||||
model = None
|
||||
@@ -162,17 +204,21 @@ def create_asset(session: Session, data: AssetCreate) -> AssetRead:
|
||||
)
|
||||
asset = Asset(**asset_data)
|
||||
session.add(asset)
|
||||
session.commit()
|
||||
session.refresh(asset)
|
||||
session.flush() # 获取 asset.id,但不提交
|
||||
|
||||
detail = None
|
||||
if model is not None and detail_in is not None:
|
||||
detail = _build_detail(data.asset_type, asset.id, detail_in)
|
||||
session.add(detail)
|
||||
session.commit()
|
||||
session.refresh(detail)
|
||||
return asset
|
||||
|
||||
return _to_read(session, asset, detail)
|
||||
|
||||
def create_asset(session: Session, data: AssetCreate) -> AssetRead:
|
||||
"""创建资产及其详情(单次事务提交)"""
|
||||
asset = _create_asset_no_commit(session, data)
|
||||
session.commit()
|
||||
session.refresh(asset)
|
||||
logger.info("创建资产 id=%s name=%s type=%s", asset.id, asset.name, asset.asset_type)
|
||||
return _to_read(session, asset, _get_detail(session, asset))
|
||||
|
||||
|
||||
def get_asset(session: Session, asset_id: int) -> AssetRead:
|
||||
@@ -183,8 +229,8 @@ def get_asset(session: Session, asset_id: int) -> AssetRead:
|
||||
return _to_read(session, asset, _get_detail(session, asset))
|
||||
|
||||
|
||||
def update_asset(session: Session, asset_id: int, data: AssetUpdate) -> AssetRead:
|
||||
"""更新资产主表及详情(仅更新传入字段)"""
|
||||
def _update_asset_no_commit(session: Session, asset_id: int, data: AssetUpdate) -> Asset:
|
||||
"""更新资产主表及详情(不 commit,由调用方统一提交)"""
|
||||
asset = session.get(Asset, asset_id)
|
||||
if not asset:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="资产不存在")
|
||||
@@ -197,7 +243,6 @@ def update_asset(session: Session, asset_id: int, data: AssetUpdate) -> AssetRea
|
||||
session.add(asset)
|
||||
|
||||
# 详情表:以更新后的 asset_type 为准
|
||||
detail = None
|
||||
item = DETAIL_MAP.get(asset.asset_type)
|
||||
if item:
|
||||
field, model = item
|
||||
@@ -209,22 +254,22 @@ def update_asset(session: Session, asset_id: int, data: AssetUpdate) -> AssetRea
|
||||
if existing:
|
||||
_apply_detail_update(asset.asset_type, existing, detail_in)
|
||||
session.add(existing)
|
||||
detail = existing
|
||||
else:
|
||||
detail = _build_detail(asset.asset_type, asset.id, detail_in)
|
||||
session.add(detail)
|
||||
else:
|
||||
detail = existing
|
||||
return asset
|
||||
|
||||
|
||||
def update_asset(session: Session, asset_id: int, data: AssetUpdate) -> AssetRead:
|
||||
"""更新资产主表及详情(单次事务提交)"""
|
||||
asset = _update_asset_no_commit(session, asset_id, data)
|
||||
session.commit()
|
||||
session.refresh(asset)
|
||||
if detail is not None:
|
||||
session.refresh(detail)
|
||||
return _to_read(session, asset, detail)
|
||||
return _to_read(session, asset, _get_detail(session, asset))
|
||||
|
||||
|
||||
def delete_asset(session: Session, asset_id: int) -> None:
|
||||
"""删除资产及其详情"""
|
||||
"""删除资产及其详情,并清理 metrics.db 中的关联监控数据(避免孤儿数据)"""
|
||||
asset = session.get(Asset, asset_id)
|
||||
if not asset:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="资产不存在")
|
||||
@@ -233,46 +278,72 @@ def delete_asset(session: Session, asset_id: int) -> None:
|
||||
session.delete(detail)
|
||||
session.delete(asset)
|
||||
session.commit()
|
||||
logger.info("删除资产 id=%s name=%s", asset_id, asset.name)
|
||||
_cleanup_metrics_for_asset(asset_id)
|
||||
|
||||
|
||||
def _cleanup_metrics_for_asset(asset_id: int) -> None:
|
||||
"""清理 metrics.db 中该资产的 MetricPoint/ServerInfo/SecurityCheck/EventLog"""
|
||||
from app.database import metrics_engine
|
||||
from app.models.monitor import EventLog, MetricPoint, SecurityCheck, ServerInfo
|
||||
|
||||
with Session(metrics_engine) as ms:
|
||||
for model in (MetricPoint, ServerInfo, SecurityCheck, EventLog):
|
||||
for row in ms.exec(select(model).where(model.asset_id == asset_id)).all():
|
||||
ms.delete(row)
|
||||
ms.commit()
|
||||
|
||||
|
||||
def export_assets(session: Session) -> dict:
|
||||
"""导出所有资产(含 detail)为 JSON,用于数据迁移/备份"""
|
||||
assets = session.exec(select(Asset)).all()
|
||||
batch = _get_details_batch(session, assets)
|
||||
result = []
|
||||
for asset in assets:
|
||||
read = _to_read(session, asset, _get_detail(session, asset))
|
||||
read = _to_read_batch(session, asset, batch)
|
||||
result.append(read.model_dump(mode="json"))
|
||||
return {
|
||||
"count": len(result),
|
||||
"exported_at": datetime.utcnow().isoformat(),
|
||||
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||||
"assets": result,
|
||||
}
|
||||
|
||||
|
||||
def import_assets(session: Session, assets_data: list) -> dict:
|
||||
"""从导出数据导入资产(按 name+asset_type+provider 去重:存在则更新,不存在则创建)"""
|
||||
"""从导出数据导入资产(按 name+asset_type+provider 去重:存在则更新,不存在则创建)
|
||||
|
||||
整个导入在单个事务中完成:全部成功才提交,任何一条失败则回滚,避免部分导入。
|
||||
"""
|
||||
created = 0
|
||||
updated = 0
|
||||
errors = []
|
||||
skip_fields = {"id", "created_at", "updated_at", "days_to_expiry", "provider_name"}
|
||||
for item in assets_data:
|
||||
try:
|
||||
payload = {k: v for k, v in item.items() if k not in skip_fields}
|
||||
existing = session.exec(
|
||||
select(Asset).where(
|
||||
Asset.name == payload.get("name"),
|
||||
Asset.asset_type == payload.get("asset_type"),
|
||||
Asset.provider == payload.get("provider"),
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
update_asset(session, existing.id, AssetUpdate(**payload))
|
||||
updated += 1
|
||||
else:
|
||||
create_asset(session, AssetCreate(**payload))
|
||||
created += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append(f"{item.get('name', '?')}: {e}")
|
||||
try:
|
||||
for item in assets_data:
|
||||
try:
|
||||
payload = {k: v for k, v in item.items() if k not in skip_fields}
|
||||
existing = session.exec(
|
||||
select(Asset).where(
|
||||
Asset.name == payload.get("name"),
|
||||
Asset.asset_type == payload.get("asset_type"),
|
||||
Asset.provider == payload.get("provider"),
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
_update_asset_no_commit(session, existing.id, AssetUpdate(**payload))
|
||||
updated += 1
|
||||
else:
|
||||
_create_asset_no_commit(session, AssetCreate(**payload))
|
||||
created += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append(f"{item.get('name', '?')}: {e}")
|
||||
if errors:
|
||||
session.rollback()
|
||||
return {"created": 0, "updated": 0, "errors": errors}
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001
|
||||
session.rollback()
|
||||
raise
|
||||
return {"created": created, "updated": updated, "errors": errors}
|
||||
|
||||
|
||||
@@ -301,15 +372,20 @@ def list_assets(
|
||||
stmt = stmt.order_by(sort_col.desc() if order == "desc" else sort_col.asc())
|
||||
|
||||
assets = session.exec(stmt).all()
|
||||
return [_to_read(session, a, _get_detail(session, a)) for a in assets]
|
||||
batch = _get_details_batch(session, assets)
|
||||
return [_to_read_batch(session, a, batch) for a in assets]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 统计
|
||||
# --------------------------------------------------------------------------- #
|
||||
def get_overview(session: Session) -> dict:
|
||||
"""资产总览:数量分布、异常数、到期预警、支出合计"""
|
||||
assets = session.exec(select(Asset)).all()
|
||||
"""资产总览:数量分布、异常数、到期预警、支出合计
|
||||
|
||||
性能优化:仅查询统计所需的列,避免加载完整 Asset 实体。
|
||||
"""
|
||||
stmt = select(Asset.asset_type, Asset.status, Asset.expiry_date, Asset.cost)
|
||||
rows = session.exec(stmt).all()
|
||||
today = date.today()
|
||||
|
||||
by_type: dict = {}
|
||||
@@ -319,22 +395,24 @@ def get_overview(session: Session) -> dict:
|
||||
year_cost = 0.0
|
||||
month_cost = 0.0
|
||||
|
||||
for a in assets:
|
||||
by_type[a.asset_type.value] = by_type.get(a.asset_type.value, 0) + 1
|
||||
by_status[a.status.value] = by_status.get(a.status.value, 0) + 1
|
||||
if a.status in (AssetStatus.STOPPED, AssetStatus.EXPIRED):
|
||||
for asset_type, status_val, expiry_date, cost in rows:
|
||||
tval = asset_type.value if hasattr(asset_type, "value") else asset_type
|
||||
sval = status_val.value if hasattr(status_val, "value") else status_val
|
||||
by_type[tval] = by_type.get(tval, 0) + 1
|
||||
by_status[sval] = by_status.get(sval, 0) + 1
|
||||
if status_val in (AssetStatus.STOPPED, AssetStatus.EXPIRED):
|
||||
abnormal += 1
|
||||
if a.expiry_date:
|
||||
days = (a.expiry_date - today).days
|
||||
if expiry_date:
|
||||
days = (expiry_date - today).days
|
||||
if 0 <= days <= 30:
|
||||
expiring_30 += 1
|
||||
if a.expiry_date.year == today.year:
|
||||
year_cost += a.cost
|
||||
if a.expiry_date.month == today.month:
|
||||
month_cost += a.cost
|
||||
if expiry_date.year == today.year:
|
||||
year_cost += cost
|
||||
if expiry_date.month == today.month:
|
||||
month_cost += cost
|
||||
|
||||
return {
|
||||
"total": len(assets),
|
||||
"total": len(rows),
|
||||
"by_type": by_type,
|
||||
"by_status": by_status,
|
||||
"abnormal_count": abnormal,
|
||||
@@ -345,13 +423,20 @@ def get_overview(session: Session) -> dict:
|
||||
|
||||
|
||||
def get_expiring(session: Session, days: int = 30) -> List[AssetRead]:
|
||||
"""N 天内到期资产列表(按剩余天数升序)"""
|
||||
"""N 天内到期资产列表(按剩余天数升序)
|
||||
|
||||
性能优化:过滤条件下推到 SQL 层,仅查询 [today, today+days] 区间内的资产。
|
||||
"""
|
||||
today = date.today()
|
||||
assets = session.exec(select(Asset).where(Asset.expiry_date.is_not(None))).all()
|
||||
result = []
|
||||
for a in assets:
|
||||
delta = (a.expiry_date - today).days
|
||||
if 0 <= delta <= days:
|
||||
result.append(_to_read(session, a, _get_detail(session, a)))
|
||||
deadline = today + timedelta(days=days)
|
||||
assets = session.exec(
|
||||
select(Asset).where(
|
||||
Asset.expiry_date.is_not(None),
|
||||
Asset.expiry_date >= today,
|
||||
Asset.expiry_date <= deadline,
|
||||
)
|
||||
).all()
|
||||
batch = _get_details_batch(session, assets)
|
||||
result = [_to_read_batch(session, a, batch) for a in assets]
|
||||
result.sort(key=lambda x: x.days_to_expiry if x.days_to_expiry is not None else 10**9)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""监控数据自动清理服务
|
||||
|
||||
按配置的保留周期清理 metrics.db 中的历史数据,防止 SQLite 无限膨胀:
|
||||
- MetricPoint: 高频时序数据,默认保留 30 天
|
||||
- SecurityCheck: 安全检查历史,默认保留 90 天
|
||||
- EventLog: 事件日志,默认保留 180 天
|
||||
|
||||
提供两种触发方式:
|
||||
1. 应用启动后由 asyncio 后台任务按 CLEANUP_INTERVAL_HOURS 周期执行
|
||||
2. 通过 API 手动触发(POST /api/monitor/cleanup)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlmodel import Session, delete, select
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.timeutils import utcnow
|
||||
from app.database import metrics_engine
|
||||
from app.models.monitor import EventLog, MetricPoint, SecurityCheck
|
||||
|
||||
logger = logging.getLogger("vps-manager.cleanup")
|
||||
|
||||
# 单批删除行数上限:避免大表单条 DELETE 长时间持有写锁,阻塞 Agent 上报
|
||||
_BATCH_SIZE = 1000
|
||||
|
||||
|
||||
def _delete_in_batches(session: Session, model, cutoff, batch_size: int = _BATCH_SIZE) -> int:
|
||||
"""分批删除指定模型的过期数据(ts < cutoff),返回总删除行数
|
||||
|
||||
SQLite 不支持 DELETE ... LIMIT,用子查询 SELECT id ... LIMIT 实现分批。
|
||||
"""
|
||||
total = 0
|
||||
while True:
|
||||
subq = select(model.id).where(model.ts < cutoff).limit(batch_size)
|
||||
ids = [row[0] if isinstance(row, tuple) else row for row in session.exec(subq).all()]
|
||||
if not ids:
|
||||
break
|
||||
session.exec(delete(model).where(model.id.in_(ids)))
|
||||
session.commit() # 每批独立提交,缩短写锁持有时间
|
||||
total += len(ids)
|
||||
if len(ids) < batch_size:
|
||||
break
|
||||
return total
|
||||
|
||||
|
||||
def cleanup_metrics(
|
||||
metrics_days: int | None = None,
|
||||
security_days: int | None = None,
|
||||
event_log_days: int | None = None,
|
||||
) -> dict:
|
||||
"""按保留天数清理过期监控数据,返回各类删除条数"""
|
||||
metrics_days = metrics_days if metrics_days is not None else settings.METRICS_RETENTION_DAYS
|
||||
security_days = security_days if security_days is not None else settings.SECURITY_RETENTION_DAYS
|
||||
event_log_days = event_log_days if event_log_days is not None else settings.EVENT_LOG_RETENTION_DAYS
|
||||
|
||||
now = utcnow()
|
||||
result = {"metric_points": 0, "security_checks": 0, "event_logs": 0}
|
||||
|
||||
with Session(metrics_engine) as session:
|
||||
if metrics_days > 0:
|
||||
cutoff = now - timedelta(days=metrics_days)
|
||||
result["metric_points"] = _delete_in_batches(session, MetricPoint, cutoff)
|
||||
|
||||
if security_days > 0:
|
||||
cutoff = now - timedelta(days=security_days)
|
||||
result["security_checks"] = _delete_in_batches(session, SecurityCheck, cutoff)
|
||||
|
||||
if event_log_days > 0:
|
||||
cutoff = now - timedelta(days=event_log_days)
|
||||
result["event_logs"] = _delete_in_batches(session, EventLog, cutoff)
|
||||
|
||||
logger.info(
|
||||
"监控数据清理完成:metric_points=%s, security_checks=%s, event_logs=%s",
|
||||
result["metric_points"],
|
||||
result["security_checks"],
|
||||
result["event_logs"],
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def cleanup_loop() -> None:
|
||||
"""后台周期清理任务(CLEANUP_INTERVAL_HOURS=0 时不启动)"""
|
||||
interval_hours = settings.CLEANUP_INTERVAL_HOURS
|
||||
if interval_hours <= 0:
|
||||
logger.info("监控数据自动清理已禁用(CLEANUP_INTERVAL_HOURS=0)")
|
||||
return
|
||||
interval_sec = interval_hours * 3600
|
||||
logger.info("监控数据自动清理已启动,间隔 %s 小时", interval_hours)
|
||||
while True:
|
||||
try:
|
||||
# 在线程池执行同步 DB 操作,避免阻塞事件循环
|
||||
await asyncio.to_thread(cleanup_metrics)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("监控数据自动清理执行失败")
|
||||
await asyncio.sleep(interval_sec)
|
||||
@@ -6,6 +6,7 @@ from fastapi import HTTPException, status
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.core import crypto
|
||||
from app.models.asset import Asset
|
||||
from app.models.provider import Provider, ProviderCategory
|
||||
from app.schemas.provider import ProviderCreate, ProviderRead, ProviderUpdate
|
||||
|
||||
@@ -73,6 +74,15 @@ def delete_provider(session: Session, provider_id: int) -> None:
|
||||
provider = session.get(Provider, provider_id)
|
||||
if not provider:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="平台不存在")
|
||||
# 数据完整性:存在关联资产时禁止删除,避免 provider_id 悬空引用
|
||||
linked = session.exec(
|
||||
select(Asset.id).where(Asset.provider_id == provider_id).limit(1)
|
||||
).first()
|
||||
if linked is not None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="该平台下仍有关联资产,请先迁移或删除相关资产",
|
||||
)
|
||||
session.delete(provider)
|
||||
session.commit()
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
检查即将到期的资产,生成提醒消息并通过通知渠道发送。
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
from datetime import date, timedelta
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
@@ -16,16 +16,21 @@ _SKIP_STATUS = {AssetStatus.CANCELLED}
|
||||
|
||||
|
||||
def get_expiring_assets(session: Session, threshold_days: int) -> list:
|
||||
"""返回 threshold_days 天内到期的资产列表 [(asset, days), ...],按剩余天数升序"""
|
||||
"""返回 threshold_days 天内到期的资产列表 [(asset, days), ...],按剩余天数升序
|
||||
|
||||
性能优化:日期范围与状态过滤均下推到 SQL 层。
|
||||
"""
|
||||
today = date.today()
|
||||
assets = session.exec(select(Asset).where(Asset.expiry_date.is_not(None))).all()
|
||||
expiring = []
|
||||
for asset in assets:
|
||||
if asset.status in _SKIP_STATUS:
|
||||
continue
|
||||
days = (asset.expiry_date - today).days
|
||||
if 0 <= days <= threshold_days:
|
||||
expiring.append((asset, days))
|
||||
deadline = today + timedelta(days=threshold_days)
|
||||
assets = session.exec(
|
||||
select(Asset).where(
|
||||
Asset.expiry_date.is_not(None),
|
||||
Asset.expiry_date >= today,
|
||||
Asset.expiry_date <= deadline,
|
||||
Asset.status.notin_(_SKIP_STATUS),
|
||||
)
|
||||
).all()
|
||||
expiring = [(asset, (asset.expiry_date - today).days) for asset in assets]
|
||||
expiring.sort(key=lambda x: x[1])
|
||||
return expiring
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
评分按检查项加权:pass 满分、warn 半分、fail/unknown 零分。
|
||||
"""
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.models.monitor import SecurityCheck
|
||||
@@ -70,8 +71,42 @@ def get_asset_security(session: Session, asset_id: int) -> dict:
|
||||
|
||||
|
||||
def get_security_overview(session: Session) -> list:
|
||||
"""所有有安全检查数据的服务器评分总览(按评分升序,风险高的在前)"""
|
||||
asset_ids = session.exec(select(SecurityCheck.asset_id).distinct()).all()
|
||||
result = [get_asset_security(session, aid) for aid in asset_ids]
|
||||
"""所有有安全检查数据的服务器评分总览(按评分升序,风险高的在前)
|
||||
|
||||
性能优化:用子查询取每个 (asset_id, check_item) 的最新 ts,仅拉取最新记录,
|
||||
避免全表扫描(历史数据量大时内存可控)。
|
||||
"""
|
||||
latest_ts = (
|
||||
select(
|
||||
SecurityCheck.asset_id,
|
||||
SecurityCheck.check_item,
|
||||
func.max(SecurityCheck.ts).label("max_ts"),
|
||||
)
|
||||
.group_by(SecurityCheck.asset_id, SecurityCheck.check_item)
|
||||
.subquery()
|
||||
)
|
||||
stmt = select(SecurityCheck).join(
|
||||
latest_ts,
|
||||
(SecurityCheck.asset_id == latest_ts.c.asset_id)
|
||||
& (SecurityCheck.check_item == latest_ts.c.check_item)
|
||||
& (SecurityCheck.ts == latest_ts.c.max_ts),
|
||||
)
|
||||
all_checks = session.exec(stmt).all()
|
||||
|
||||
# 按 asset_id 分组
|
||||
latest_by_asset: dict = {}
|
||||
for check in all_checks:
|
||||
latest_by_asset.setdefault(check.asset_id, []).append(check)
|
||||
|
||||
result = []
|
||||
for asset_id, checks in latest_by_asset.items():
|
||||
score = compute_security_score(checks)
|
||||
result.append({
|
||||
"asset_id": asset_id,
|
||||
"score": score,
|
||||
"level": score_level(score),
|
||||
"checks_count": len(checks),
|
||||
"suggestions": [c.suggestion for c in checks if c.suggestion],
|
||||
})
|
||||
result.sort(key=lambda x: (x["score"] is None, x["score"] if x["score"] is not None else 0))
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""SSL 监控服务:站点证书探测 + 子域名管理
|
||||
|
||||
- probe_site_cert: 通过 ssl socket 探测站点证书信息(不校验证书链,仅读取到期信息)
|
||||
- create/check/check_all: 站点证书的创建与刷新
|
||||
- 子域名 CRUD:list/create/update/delete
|
||||
"""
|
||||
|
||||
import socket
|
||||
import ssl
|
||||
from datetime import date, datetime
|
||||
from typing import Optional
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.core.timeutils import utcnow
|
||||
from app.models.asset import Asset
|
||||
from app.models.ssl import SiteCert, Subdomain
|
||||
|
||||
PROBE_TIMEOUT = 8 # 探测连接超时(秒)
|
||||
EXPIRING_THRESHOLD = 30 # 到期提醒阈值(天)
|
||||
|
||||
|
||||
def days_until(d: Optional[date]) -> Optional[int]:
|
||||
"""计算距今天数(负数为已过期)"""
|
||||
if d is None:
|
||||
return None
|
||||
return (d - date.today()).days
|
||||
|
||||
|
||||
def _judge_status(days: Optional[int]) -> str:
|
||||
"""按剩余天数定级:expired / expiring / valid"""
|
||||
if days is None:
|
||||
return "unknown"
|
||||
if days < 0:
|
||||
return "expired"
|
||||
if days <= EXPIRING_THRESHOLD:
|
||||
return "expiring"
|
||||
return "valid"
|
||||
|
||||
|
||||
def probe_site_cert(hostname: str, port: int = 443) -> dict:
|
||||
"""探测目标站点证书信息(探测失败时抛异常)
|
||||
|
||||
返回字段:subject_cn / issuer / valid_from / valid_to / fingerprint / san_list
|
||||
"""
|
||||
ctx = ssl.create_default_context()
|
||||
# 不校验证书链:即使证书已过期/自签名也能读到到期信息
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
with socket.create_connection((hostname, port), timeout=PROBE_TIMEOUT) as sock:
|
||||
with ctx.wrap_socket(sock, server_hostname=hostname) as ssock:
|
||||
der = ssock.getpeercert(binary_form=True)
|
||||
cert = x509.load_der_x509_certificate(der)
|
||||
try:
|
||||
san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName)
|
||||
san_list = san.value.get_values_for_type(x509.DNSName)
|
||||
except x509.ExtensionNotFound:
|
||||
san_list = []
|
||||
return {
|
||||
"subject_cn": cert.subject.rfc4514_string() or None,
|
||||
"issuer": cert.issuer.rfc4514_string() or None,
|
||||
"valid_from": cert.not_valid_before_utc.date(),
|
||||
"valid_to": cert.not_valid_after_utc.date(),
|
||||
"fingerprint": cert.fingerprint(hashes.SHA256()).hex(),
|
||||
"san_list": san_list,
|
||||
}
|
||||
|
||||
|
||||
def _to_dict(cert: SiteCert, asset_name: Optional[str] = None) -> dict:
|
||||
"""SiteCert 模型转 API 返回结构(含动态计算的剩余天数)"""
|
||||
days = days_until(cert.valid_to)
|
||||
return {
|
||||
"id": cert.id,
|
||||
"hostname": cert.hostname,
|
||||
"port": cert.port,
|
||||
"asset_id": cert.asset_id,
|
||||
"asset_name": asset_name,
|
||||
"issuer": cert.issuer,
|
||||
"subject_cn": cert.subject_cn,
|
||||
"valid_from": cert.valid_from.isoformat() if cert.valid_from else None,
|
||||
"valid_to": cert.valid_to.isoformat() if cert.valid_to else None,
|
||||
"days_to_expiry": days,
|
||||
"status": cert.status,
|
||||
"error": cert.error,
|
||||
"last_checked_at": cert.last_checked_at.isoformat() if cert.last_checked_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------- 子域名 CRUD ----------------
|
||||
|
||||
def list_subdomains(session: Session, asset_id: Optional[int] = None) -> list:
|
||||
stmt = select(Subdomain).order_by(Subdomain.host)
|
||||
if asset_id is not None:
|
||||
stmt = stmt.where(Subdomain.asset_id == asset_id)
|
||||
subs = session.exec(stmt).all()
|
||||
# 带上所属域名,便于前端展示全名
|
||||
assets = session.exec(select(Asset).where(Asset.id.in_({s.asset_id for s in subs}))).all()
|
||||
name_map = {a.id: a.name for a in assets}
|
||||
return [
|
||||
{
|
||||
"id": s.id,
|
||||
"asset_id": s.asset_id,
|
||||
"asset_name": name_map.get(s.asset_id),
|
||||
"host": s.host,
|
||||
"record_type": s.record_type,
|
||||
"record_value": s.record_value,
|
||||
"is_active": s.is_active,
|
||||
"note": s.note,
|
||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||
}
|
||||
for s in subs
|
||||
]
|
||||
|
||||
|
||||
def create_subdomain(session: Session, data: dict) -> Subdomain:
|
||||
sub = Subdomain(
|
||||
asset_id=data["asset_id"],
|
||||
host=data["host"].strip().lower(),
|
||||
record_type=data.get("record_type"),
|
||||
record_value=data.get("record_value"),
|
||||
is_active=data.get("is_active", True),
|
||||
note=data.get("note"),
|
||||
)
|
||||
session.add(sub)
|
||||
session.commit()
|
||||
session.refresh(sub)
|
||||
return sub
|
||||
|
||||
|
||||
def update_subdomain(session: Session, sub_id: int, data: dict) -> Subdomain:
|
||||
sub = session.get(Subdomain, sub_id)
|
||||
if not sub:
|
||||
raise ValueError(f"子域名记录不存在(id={sub_id})")
|
||||
if "host" in data and data["host"]:
|
||||
sub.host = data["host"].strip().lower()
|
||||
for key in ("record_type", "record_value", "note"):
|
||||
if key in data:
|
||||
setattr(sub, key, data[key])
|
||||
if "is_active" in data:
|
||||
sub.is_active = bool(data["is_active"])
|
||||
session.add(sub)
|
||||
session.commit()
|
||||
session.refresh(sub)
|
||||
return sub
|
||||
|
||||
|
||||
def delete_subdomain(session: Session, sub_id: int) -> None:
|
||||
sub = session.get(Subdomain, sub_id)
|
||||
if not sub:
|
||||
raise ValueError(f"子域名记录不存在(id={sub_id})")
|
||||
session.delete(sub)
|
||||
session.commit()
|
||||
|
||||
|
||||
# ---------------- 站点证书监控 ----------------
|
||||
|
||||
def _apply_probe(cert: SiteCert, info: dict) -> SiteCert:
|
||||
"""把探测结果写入模型并定级"""
|
||||
cert.subject_cn = info["subject_cn"]
|
||||
cert.issuer = info["issuer"]
|
||||
cert.valid_from = info["valid_from"]
|
||||
cert.valid_to = info["valid_to"]
|
||||
cert.fingerprint = info["fingerprint"]
|
||||
cert.error = None
|
||||
cert.status = _judge_status(days_until(info["valid_to"]))
|
||||
cert.last_checked_at = utcnow()
|
||||
return cert
|
||||
|
||||
|
||||
def check_one(session: Session, cert: SiteCert) -> SiteCert:
|
||||
"""重新探测单条证书记录(失败则标记 error,保留旧到期信息)"""
|
||||
try:
|
||||
info = probe_site_cert(cert.hostname, cert.port)
|
||||
_apply_probe(cert, info)
|
||||
except Exception as e: # noqa: BLE001
|
||||
cert.status = "error"
|
||||
cert.error = str(e)[:200]
|
||||
cert.last_checked_at = utcnow()
|
||||
session.add(cert)
|
||||
session.commit()
|
||||
session.refresh(cert)
|
||||
return cert
|
||||
|
||||
|
||||
def create_site_cert(session: Session, hostname: str, port: int = 443, asset_id: Optional[int] = None) -> SiteCert:
|
||||
"""创建探测目标并立即探测一次;hostname+port 已存在则复用并刷新"""
|
||||
hostname = hostname.strip().lower()
|
||||
existing = session.exec(
|
||||
select(SiteCert).where(SiteCert.hostname == hostname, SiteCert.port == port)
|
||||
).first()
|
||||
if existing:
|
||||
if asset_id is not None:
|
||||
existing.asset_id = asset_id
|
||||
return check_one(session, existing)
|
||||
cert = SiteCert(hostname=hostname, port=port, asset_id=asset_id)
|
||||
session.add(cert)
|
||||
session.commit()
|
||||
session.refresh(cert)
|
||||
return check_one(session, cert)
|
||||
|
||||
|
||||
def list_site_certs(session: Session, status: Optional[str] = None, asset_id: Optional[int] = None) -> list:
|
||||
stmt = select(SiteCert)
|
||||
if status:
|
||||
stmt = stmt.where(SiteCert.status == status)
|
||||
if asset_id is not None:
|
||||
stmt = stmt.where(SiteCert.asset_id == asset_id)
|
||||
certs = session.exec(stmt.order_by(SiteCert.id)).all()
|
||||
ids = {c.asset_id for c in certs if c.asset_id}
|
||||
assets = session.exec(select(Asset).where(Asset.id.in_(ids))).all() if ids else []
|
||||
name_map = {a.id: a.name for a in assets}
|
||||
return [_to_dict(c, name_map.get(c.asset_id)) for c in certs]
|
||||
|
||||
|
||||
def delete_site_cert(session: Session, cert_id: int) -> None:
|
||||
cert = session.get(SiteCert, cert_id)
|
||||
if not cert:
|
||||
raise ValueError(f"证书监控记录不存在(id={cert_id})")
|
||||
session.delete(cert)
|
||||
session.commit()
|
||||
|
||||
|
||||
def check_all_site_certs(session: Session) -> dict:
|
||||
"""全量刷新所有站点证书,返回统计与异常清单"""
|
||||
certs = session.exec(select(SiteCert)).all()
|
||||
stats = {"total": len(certs), "ok": 0, "error": 0, "expiring": 0, "expired": 0}
|
||||
problems = []
|
||||
for cert in certs:
|
||||
check_one(session, cert)
|
||||
if cert.status == "error":
|
||||
stats["error"] += 1
|
||||
problems.append({"hostname": cert.hostname, "detail": cert.error})
|
||||
elif cert.status == "expired":
|
||||
stats["expired"] += 1
|
||||
problems.append(
|
||||
{"hostname": cert.hostname, "detail": f"证书已过期 {abs(days_until(cert.valid_to))} 天"}
|
||||
)
|
||||
elif cert.status == "expiring":
|
||||
stats["expiring"] += 1
|
||||
problems.append(
|
||||
{"hostname": cert.hostname, "detail": f"{days_until(cert.valid_to)} 天后到期"}
|
||||
)
|
||||
else:
|
||||
stats["ok"] += 1
|
||||
return {"stats": stats, "problems": problems}
|
||||
|
||||
|
||||
def build_cert_message(cert: SiteCert) -> str:
|
||||
"""生成单条证书提醒文案"""
|
||||
days = days_until(cert.valid_to)
|
||||
if days is None:
|
||||
return f"- {cert.hostname}:证书信息未知"
|
||||
if days < 0:
|
||||
return f"- ⚠️ {cert.hostname}:证书已过期 {abs(days)} 天"
|
||||
return f"- {cert.hostname}:{days} 天后到期({cert.valid_to})"
|
||||
@@ -6,15 +6,18 @@
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlmodel import Session, select
|
||||
|
||||
logger = logging.getLogger("vps-manager.sync")
|
||||
|
||||
from app.adapters import registry
|
||||
from app.adapters.base import BaseAdapter
|
||||
from app.core import crypto
|
||||
from app.core.timeutils import utcnow
|
||||
from app.models.asset import (
|
||||
AIAccount,
|
||||
Asset,
|
||||
@@ -130,8 +133,7 @@ def _sync_vps(session: Session, provider: Provider, adapter: BaseAdapter) -> dic
|
||||
currency=vps.currency,
|
||||
)
|
||||
session.add(asset)
|
||||
session.commit()
|
||||
session.refresh(asset)
|
||||
session.flush() # 获取 asset.id,统一在循环外提交
|
||||
session.add(
|
||||
VPSDetail(
|
||||
asset_id=asset.id,
|
||||
@@ -176,8 +178,7 @@ def _sync_domains(session: Session, provider: Provider, adapter: BaseAdapter) ->
|
||||
expiry_date=dom.expiry_date,
|
||||
)
|
||||
session.add(asset)
|
||||
session.commit()
|
||||
session.refresh(asset)
|
||||
session.flush() # 获取 asset.id,统一在循环外提交
|
||||
session.add(
|
||||
DomainDetail(
|
||||
asset_id=asset.id,
|
||||
@@ -219,10 +220,11 @@ def sync_provider(session: Session, provider_id: int) -> dict:
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["account_error"] = str(e)
|
||||
|
||||
provider.last_synced_at = datetime.utcnow()
|
||||
provider.last_synced_at = utcnow()
|
||||
session.add(provider)
|
||||
session.commit()
|
||||
result["last_synced_at"] = provider.last_synced_at.isoformat()
|
||||
logger.info("同步平台 provider=%s result=%s", provider.slug, result)
|
||||
return result
|
||||
|
||||
|
||||
@@ -262,7 +264,7 @@ def refresh_ai_balance(session: Session, asset_id: int) -> dict:
|
||||
detail=f"无法确定 AI 适配器({ai.provider})",
|
||||
)
|
||||
|
||||
api_key = crypto.decrypt(ai.api_key_encrypted) or ai.api_key
|
||||
api_key = crypto.decrypt(ai.api_key_encrypted)
|
||||
if not api_key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="未配置 API Key"
|
||||
@@ -286,7 +288,7 @@ def refresh_ai_balance(session: Session, asset_id: int) -> dict:
|
||||
if acc.balance is not None:
|
||||
ai.balance = acc.balance
|
||||
ai.currency = acc.currency
|
||||
ai.last_synced_at = datetime.utcnow()
|
||||
ai.last_synced_at = utcnow()
|
||||
session.add(ai)
|
||||
session.commit()
|
||||
session.refresh(ai)
|
||||
@@ -320,5 +322,5 @@ def sync_all_ai_balances(session: Session) -> dict:
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"errors": errors,
|
||||
"synced_at": datetime.utcnow().isoformat(),
|
||||
"synced_at": utcnow().isoformat(),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user