Files
vps-manager/app/services/sync_service.py
T
gouki abd2fa8206 feat(renewal): 提醒前核对平台真实状态,消除手工数据导致的误报
容错策略:核对失败/无可用通道时标记「未核对」并继续按本地数据提醒,
避免因核对链路故障漏掉真正到期的资产;已停止资产仅在 auto_renew=true
时提醒(停止实例不等于停止扣费)。新增 WHMCS 面板只读适配器,
用于 TotHost 这类没有开放 API 的平台。
2026-10-25 23:05:39 +00:00

614 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""云厂商同步服务
将适配器返回的标准化资产(NormalizedVPS / NormalizedDomain / AccountInfo)
写入或更新到资产库。以 (provider_id, external_id) 作为去重键,
已存在则更新状态/详情,不存在则新建资产。
凭证层级:API 配置存在账号(Account.api_config_encrypted)上,
同步按账号维度进行(test_account/sync_account);同步产出的资产
自动挂到该账号名下(Asset.account)。平台不再持有凭证。
"""
import json
import logging
from typing import Optional
from fastapi import HTTPException, status
from sqlmodel import Session, select
logger = logging.getLogger("vps-manager.sync")
from app.adapters import registry
from app.adapters.base import BaseAdapter, cloud_error_text, parse_iso_date
from app.core import crypto
from app.core.timeutils import utcnow
from app.models.asset import (
AIAccount,
Account,
Asset,
AssetStatus,
AssetType,
DomainDetail,
VPSDetail,
)
from app.models.credential import Credential
from app.models.provider import Provider
_VALID_STATUS = {s.value for s in AssetStatus}
def _load_account_config(account: Account) -> dict:
"""解密账号的 API 配置 JSON"""
plain = crypto.decrypt(account.api_config_encrypted)
if not plain:
return {}
try:
return json.loads(plain)
except (json.JSONDecodeError, TypeError):
return {}
def _credential_login_config(session: Session, account: Account) -> dict:
"""从凭据表取面板登录所需的用户名/密码(凭据表是唯一事实源)"""
if not account.credential_id:
return {}
cred = session.get(Credential, account.credential_id)
if not cred or not cred.password_encrypted:
return {}
password = crypto.decrypt(cred.password_encrypted)
if not password:
return {}
return {"username": cred.username or account.name, "password": password}
def _find_provider_by_platform(session: Session, platform: str) -> Provider:
"""按账号的 platform(slug 或名称)定位平台"""
provider = session.exec(select(Provider).where(Provider.slug == platform)).first()
if not provider:
provider = session.exec(select(Provider).where(Provider.name == platform)).first()
if not provider:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"账号所属平台不存在:{platform}(请先在账号编辑中选择有效平台)",
)
return provider
def _get_provider(session: Session, provider_id: int) -> Provider:
provider = session.get(Provider, provider_id)
if not provider:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="平台不存在")
return provider
def _build_adapter(provider: Provider, config: dict) -> BaseAdapter:
if not provider.sdk_type:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="该平台未配置 SDK 类型(sdk_type)"
)
if not registry.is_supported(provider.sdk_type):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"暂不支持的 SDK 类型:{provider.sdk_type}",
)
return registry.get_adapter(provider.sdk_type, config)
def _get_account(session: Session, account_id: int) -> Account:
account = session.get(Account, account_id)
if not account:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="账号不存在")
return account
def _build_adapter_for_account(session: Session, account: Account):
"""按账号构建适配器:平台定 sdk_type,账号提供凭证"""
if not account.platform:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="该账号未指定所属平台,无法确定 SDK 类型",
)
provider = _find_provider_by_platform(session, account.platform)
config = _load_account_config(account)
if provider.sdk_type == "whmcs-panel":
# 面板登录类适配器不重复存密码:用户名/密码取自关联凭据,
# base_url 取自平台的管理面板地址。
config = {**_credential_login_config(session, account), **config}
config.setdefault("base_url", provider.console_url or provider.website or "")
if not config.get("password"):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="该平台通过账单面板登录核对,请为账号关联带密码的登录凭据",
)
adapter = _build_adapter(provider, config)
return provider, adapter
def _norm_status(raw: Optional[str]) -> AssetStatus:
return AssetStatus(raw) if raw in _VALID_STATUS else AssetStatus.UNKNOWN
def _missing_config(adapter: BaseAdapter) -> list:
"""检查适配器所需凭证是否已配置"""
return [f for f in adapter.required_config if not adapter.config.get(f)]
def test_provider(session: Session, provider_id: int) -> dict:
"""(兼容入口)测试平台连接:凭证已下沉到账号,自动找该平台第一个配了凭证的账号"""
provider = _get_provider(session, provider_id)
account = _first_account_with_config(session, provider)
if not account:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="凭证已改为在账号上配置:请先在该平台的账号管理中新建账号并填写 API 配置",
)
return test_account(session, account.id)
def _account_has_credentials(session: Session, account: Account, provider: Provider) -> bool:
"""账号是否具备该平台可用的同步凭证
普通云厂商看 api_config_encrypted;WHMCS 面板类平台看关联登录凭据。
"""
if account.api_config_encrypted:
return True
if provider.sdk_type == "whmcs-panel":
return bool(_credential_login_config(session, account))
return False
def _first_account_with_config(session: Session, provider: Provider):
"""找该平台下第一个具备可用凭证的账号(兼容旧入口)"""
accounts = session.exec(
select(Account)
.where(Account.platform == provider.slug)
.order_by(Account.id.asc())
).all()
for account in accounts:
if _account_has_credentials(session, account, provider):
return account
return None
def test_account(session: Session, account_id: int) -> dict:
"""测试账号凭证有效性(按账号 platform 定 SDK,凭证取自账号)"""
account = _get_account(session, account_id)
provider, adapter = _build_adapter_for_account(session, account)
base = {"capabilities": adapter.capabilities(), "sdk_type": provider.sdk_type, "account": account.name}
missing = _missing_config(adapter)
if missing:
return {
"ok": False,
"message": f"缺少凭证配置:{', '.join(missing)}(请在账号编辑里填写 API 配置)",
**base,
}
result = adapter.test_connection()
result.update(base)
return result
def _find_asset(session: Session, provider_id: int, external_id: str, asset_type: AssetType):
return session.exec(
select(Asset).where(
Asset.provider_id == provider_id,
Asset.external_id == external_id,
Asset.asset_type == asset_type,
)
).first()
def _find_unlinked_vps(session: Session, provider: Provider, vps) -> Optional[Asset]:
"""在手工登记、尚无 external_id 的同平台 VPS 中按 IP/名称找对应项
手工登记资产没有云端实例 ID,直接同步会重复建一条;
能唯一对上时补写 external_id,把手工台账接进自动同步。
"""
candidates = session.exec(
select(Asset).where(
Asset.provider_id == provider.id,
Asset.asset_type == AssetType.VPS,
Asset.external_id.is_(None),
)
).all()
vps_ip = (vps.ip_address or "").strip()
vps_name = (vps.name or "").strip().lower()
for asset in candidates:
detail = session.exec(
select(VPSDetail).where(VPSDetail.asset_id == asset.id)
).first()
if vps_ip and detail and (detail.ip_address or "").strip() == vps_ip:
return asset
if vps_name and (asset.name or "").strip().lower() == vps_name:
return asset
return None
def _sync_vps(session: Session, provider: Provider, adapter: BaseAdapter, account_id: Optional[int] = None) -> dict:
created = updated = 0
for vps in adapter.list_vps():
# 预付费实例(阿里云/腾讯云包年包月)带到期日,后付费平台为 None。
# 漏了这一步,云端同步进来的 VPS 永远进不了续费提醒——而那是台账的核心用途。
expiry = parse_iso_date(vps.expiry_date)
existing = _find_asset(session, provider.id, vps.external_id, AssetType.VPS)
if not existing:
existing = _find_unlinked_vps(session, provider, vps)
if existing:
existing.external_id = vps.external_id
if existing:
existing.name = vps.name or existing.name
existing.status = _norm_status(vps.status)
if vps.monthly_cost is not None:
existing.cost = vps.monthly_cost
existing.currency = vps.currency
if expiry:
existing.expiry_date = expiry
if account_id:
existing.account_id = account_id
session.add(existing)
detail = session.exec(
select(VPSDetail).where(VPSDetail.asset_id == existing.id)
).first()
if detail:
detail.ip_address = vps.ip_address or detail.ip_address
detail.region = vps.region or detail.region
detail.os = vps.os or detail.os
detail.cpu_cores = vps.cpu_cores or detail.cpu_cores
detail.memory_gb = vps.memory_gb or detail.memory_gb
detail.disk_gb = vps.disk_gb or detail.disk_gb
session.add(detail)
updated += 1
else:
asset = Asset(
name=vps.name or vps.external_id,
asset_type=AssetType.VPS,
provider=provider.slug,
provider_id=provider.id,
external_id=vps.external_id,
account_id=account_id,
status=_norm_status(vps.status),
cost=vps.monthly_cost or 0,
currency=vps.currency,
expiry_date=expiry,
)
session.add(asset)
session.flush() # 获取 asset.id,统一在循环外提交
session.add(
VPSDetail(
asset_id=asset.id,
ip_address=vps.ip_address or "",
region=vps.region,
os=vps.os,
cpu_cores=vps.cpu_cores or 1,
memory_gb=vps.memory_gb or 1,
disk_gb=vps.disk_gb or 20,
)
)
created += 1
session.commit()
return {"created": created, "updated": updated}
def _sync_domains(session: Session, provider: Provider, adapter: BaseAdapter, account_id: Optional[int] = None) -> dict:
created = updated = 0
for dom in adapter.list_domains():
# 适配器契约上 expiry_date 已是 date;这里再归一一次,
# 防止旧适配器/自定义适配器返回字符串时整个 commit 抛 TypeError
expiry = parse_iso_date(dom.expiry_date)
existing = _find_asset(session, provider.id, dom.external_id, AssetType.DOMAIN)
if existing:
existing.status = _norm_status(dom.status)
if expiry:
existing.expiry_date = expiry
if account_id:
existing.account_id = account_id
session.add(existing)
detail = session.exec(
select(DomainDetail).where(DomainDetail.asset_id == existing.id)
).first()
if detail:
detail.domain_name = dom.domain_name or detail.domain_name
detail.registrar = dom.registrar or detail.registrar
session.add(detail)
updated += 1
else:
asset = Asset(
name=dom.domain_name,
asset_type=AssetType.DOMAIN,
provider=provider.slug,
provider_id=provider.id,
external_id=dom.external_id,
account_id=account_id,
status=_norm_status(dom.status),
expiry_date=expiry,
)
session.add(asset)
session.flush() # 获取 asset.id,统一在循环外提交
session.add(
DomainDetail(
asset_id=asset.id,
domain_name=dom.domain_name,
registrar=dom.registrar or provider.slug,
)
)
created += 1
session.commit()
return {"created": created, "updated": updated}
def sync_provider(session: Session, provider_id: int) -> dict:
"""(兼容入口)同步平台资产:凭证已下沉到账号,自动找该平台第一个配了凭证的账号"""
provider = _get_provider(session, provider_id)
account = _first_account_with_config(session, provider)
if not account:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="凭证已改为在账号上配置:请先在该平台的账号管理中新建账号并填写 API 配置",
)
return sync_account(session, account.id)
def sync_account(session: Session, account_id: int) -> dict:
"""同步账号资产到本地库(凭证取自账号,同步产出自动挂到该账号名下)"""
account = _get_account(session, account_id)
provider, adapter = _build_adapter_for_account(session, account)
missing = _missing_config(adapter)
if missing:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"缺少凭证配置:{', '.join(missing)}(请先在账号编辑里填写 API 配置)",
)
caps = adapter.capabilities()
result = {"provider": provider.slug, "sdk_type": provider.sdk_type, "account": account.name}
if caps["list_vps"]:
try:
result["vps"] = _sync_vps(session, provider, adapter, account.id)
except Exception as e: # noqa: BLE001
# 写库失败会让 Session 进入 pending-rollback 状态,必须先回滚才能继续,
# 否则下面更新 last_synced_at 的 commit 会抛 PendingRollbackError → 接口 500
session.rollback()
result["vps_error"] = cloud_error_text(e)
if caps["list_domains"]:
try:
result["domains"] = _sync_domains(session, provider, adapter, account.id)
except Exception as e: # noqa: BLE001
session.rollback()
result["domains_error"] = cloud_error_text(e)
if caps["get_account"]:
try:
result["account_info"] = adapter.get_account().to_dict()
except Exception as e: # noqa: BLE001
result["account_info_error"] = cloud_error_text(e)
account.last_synced_at = utcnow()
session.add(account)
session.commit()
result["last_synced_at"] = account.last_synced_at.isoformat()
logger.info("同步账号 account=%s provider=%s result=%s", account.name, provider.slug, result)
return result
# AI 服务商 slug -> sdk_type 映射(Provider 未配 sdk_type 时的回退推断)
_AI_SDK_MAP = {
"openai": "openai-api",
"deepseek": "deepseek-api",
"kimi": "moonshot-api",
"moonshot": "moonshot-api",
"minimax": "minimax-api",
}
class BalanceUnsupported(HTTPException):
"""该平台没有可用的余额查询接口
与「查询失败」区分开:sync_all_ai_balances 把它计入 skipped 而不是 failed,
否则每次「同步 AI 余额」都会把这类平台报成错误。
"""
def __init__(self, sdk_type: str):
super().__init__(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"{sdk_type} 暂不支持余额查询",
)
def refresh_ai_balance(session: Session, asset_id: int) -> dict:
"""刷新 AI 账号余额:解密 api_key → 适配器 get_account → 更新余额"""
asset = session.get(Asset, asset_id)
if not asset or asset.asset_type != AssetType.AI_AGENT:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="非 AI 账号资产"
)
ai = session.exec(select(AIAccount).where(AIAccount.asset_id == asset_id)).first()
if not ai:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="AI 账号详情不存在"
)
sdk_type = None
if asset.provider_id:
provider = session.get(Provider, asset.provider_id)
if provider:
sdk_type = provider.sdk_type
if not sdk_type:
sdk_type = _AI_SDK_MAP.get((ai.provider or "").lower())
if not sdk_type or not registry.is_supported(sdk_type):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"无法确定 AI 适配器({ai.provider})",
)
api_key = crypto.decrypt(ai.api_key_encrypted)
if not api_key:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="未配置 API Key"
)
adapter = registry.get_adapter(sdk_type, {"api_key": api_key})
if not adapter.capabilities().get("get_account"):
raise BalanceUnsupported(sdk_type)
try:
acc = adapter.get_account()
except NotImplementedError as e:
raise BalanceUnsupported(sdk_type) from e
except HTTPException:
raise
except Exception as e: # noqa: BLE001
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY, detail=f"查询余额失败:{cloud_error_text(e)}"
) from e
if acc.balance is not None:
ai.balance = acc.balance
ai.currency = acc.currency
ai.last_synced_at = utcnow()
session.add(ai)
session.commit()
session.refresh(ai)
return {
"balance": ai.balance,
"currency": ai.currency,
"last_synced_at": ai.last_synced_at.isoformat(),
}
def sync_all_ai_balances(session: Session, max_workers: int = 5) -> dict:
"""并发刷新所有 AI 账号余额(单个失败不中断整体)
每个账号需调用外部 API(单次最长 30s),串行时总耗时随账号数线性增长;
改为线程池并发后显著提速。注意:SQLite Session 不能跨线程共享,
每个 worker 使用独立 Session(WAL 模式下多连接读写安全)。
"""
from concurrent.futures import ThreadPoolExecutor
from app.database import assets_engine
ai_assets = session.exec(
select(Asset).where(Asset.asset_type == AssetType.AI_AGENT)
).all()
asset_ids = [(a.id, a.name) for a in ai_assets]
def _refresh_one(asset_id: int):
with Session(assets_engine) as worker_session:
refresh_ai_balance(worker_session, asset_id)
success = 0
failed = 0
skipped = 0
errors = []
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(_refresh_one, aid): name for aid, name in asset_ids}
for future in futures:
name = futures[future]
try:
future.result()
success += 1
except BalanceUnsupported:
# 平台本身没余额接口,不算失败(否则每次都报一堆红)
skipped += 1
except HTTPException as e:
failed += 1
errors.append(f"{name}: {e.detail}")
except Exception as e: # noqa: BLE001
failed += 1
errors.append(f"{name}: {cloud_error_text(e)}")
return {
"total": len(asset_ids),
"success": success,
"failed": failed,
"skipped": skipped,
"errors": errors,
"synced_at": utcnow().isoformat(),
}
def sync_all_assets(session: Session, max_workers: int = 5) -> dict:
"""并发同步所有已配凭证账号的云端资产(VPS / 域名)
与 sync_all_ai_balances 同构:单个账号失败不中断整体,每个 worker 用独立
Session(SQLite Session 不能跨线程共享,WAL 模式下多连接读写安全)。
未配凭证或平台无适配器的账号计入 skipped 而非 failed——否则每次定时同步
都会把手工登记的账号报成一堆错误。WHMCS 面板类账号也跳过:续费检查会
按需登录核对,避免全量同步重命名手写资产或为历史服务新建台账。
"""
from concurrent.futures import ThreadPoolExecutor
from app.database import assets_engine
accounts = session.exec(select(Account)).all()
targets = []
skipped_no_config = []
skipped_no_adapter = []
skipped_panel = []
for acc in accounts:
label = f"{acc.platform}/{acc.name}"
try:
provider = _find_provider_by_platform(session, acc.platform)
except HTTPException:
skipped_no_adapter.append(f"{label}(平台不存在)")
continue
if not provider.sdk_type or not registry.is_supported(provider.sdk_type):
skipped_no_adapter.append(f"{label}({provider.sdk_type or '未配 sdk_type'})")
continue
if provider.sdk_type == "whmcs-panel":
# 面板类平台由续费检查按需核对:定时全量同步会把手写资产名改成面板名、
# 并可能为面板里的历史服务新建台账,副作用大于收益。
skipped_panel.append(label)
continue
if not _account_has_credentials(session, acc, provider):
skipped_no_config.append(label)
continue
targets.append((acc.id, label))
def _sync_one(account_id: int) -> dict:
with Session(assets_engine) as worker_session:
return sync_account(worker_session, account_id)
success = 0
failed = 0
errors = []
created = updated = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(_sync_one, aid): label for aid, label in targets}
for future in futures:
label = futures[future]
try:
result = future.result()
created += (result.get("vps", {}) or {}).get("created", 0)
created += (result.get("domains", {}) or {}).get("created", 0)
updated += (result.get("vps", {}) or {}).get("updated", 0)
updated += (result.get("domains", {}) or {}).get("updated", 0)
# 局部失败(某一类资产同步抛错)不整体算失败,但必须上报
sub_error_keys = [
k for k in ("vps_error", "domains_error", "account_info_error") if result.get(k)
]
for key in sub_error_keys:
errors.append(f"{label} {key}: {result[key]}")
# 但「每一项尝试都失败」不能算成功:凭证过期时三类全 401,
# 若仍计成功,汇总行会显示「成功 1」,用户以为拉到了数据、实际一条没有。
attempted_any = any(result.get(k) is not None for k in ("vps", "domains", "account_info"))
if sub_error_keys and not attempted_any:
failed += 1
else:
success += 1
except HTTPException as e:
failed += 1
errors.append(f"{label}: {e.detail}")
except Exception as e: # noqa: BLE001
failed += 1
errors.append(f"{label}: {cloud_error_text(e)}")
return {
"total": len(targets),
"success": success,
"failed": failed,
"created": created,
"updated": updated,
"skipped_no_config": len(skipped_no_config),
"skipped_no_adapter": len(skipped_no_adapter),
"skipped_panel": len(skipped_panel),
"errors": errors,
"synced_at": utcnow().isoformat(),
}