Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b48e33fc6b | ||
|
|
d9f12685fa | ||
|
|
b3367eedaa | ||
|
|
e6a4988cf9 | ||
|
|
becf2c7c52 | ||
|
|
d421a9da72 | ||
|
|
f4389612d1 | ||
|
|
ac5a332bce | ||
|
|
9ba89ef2d1 | ||
|
|
b1023eb783 | ||
|
|
2f287ef538 | ||
|
|
f5cbbe9eef | ||
|
|
5fc56143ee | ||
|
|
d521585bf2 | ||
|
|
35e47bf0a1 | ||
|
|
f52e42d9c0 | ||
|
|
1ab56ad0b1 | ||
|
|
1889c8ebba | ||
|
|
910a05107a | ||
|
|
857d6f2562 | ||
|
|
5ce12771b9 | ||
|
|
4c152230aa | ||
|
|
9ce46cd883 | ||
|
|
140ac20281 | ||
|
|
9afc2d5a6c | ||
|
|
8585d101d5 | ||
|
|
62eb9578fd | ||
|
|
848657219e |
@@ -1,12 +1,16 @@
|
||||
# 构建阶段:安装依赖并打包 H5
|
||||
FROM node:18-alpine AS build
|
||||
|
||||
ARG APP_GIT_SHA=unknown
|
||||
ARG APP_BUILD_TIME=unknown
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
RUN printf '{"gitSha":"%s","buildTime":"%s"}\n' "$APP_GIT_SHA" "$APP_BUILD_TIME" > public/version.json
|
||||
RUN npm run build
|
||||
|
||||
# 运行阶段:nginx 托管静态资源并反向代理 /api 到后端
|
||||
@@ -14,6 +18,11 @@ RUN npm run build
|
||||
# 新版 nginx(>=1.31) 用 pwrite 写 pid 文件会被拦导致致命退出;1.28 用 write() 可正常启动。
|
||||
FROM nginx:1.28-alpine
|
||||
|
||||
ARG APP_GIT_SHA=unknown
|
||||
ARG APP_BUILD_TIME=unknown
|
||||
LABEL org.opencontainers.image.revision=${APP_GIT_SHA} \
|
||||
org.opencontainers.image.created=${APP_BUILD_TIME}
|
||||
|
||||
COPY --from=build /app/dist /usr/share/nginx/html
|
||||
# 覆盖 nginx 默认主配置(含唯一可写的 pid /tmp/nginx.pid,规避受限容器内 /run 不可写导致反复重启)
|
||||
COPY nginx.conf /etc/nginx/nginx.conf
|
||||
|
||||
@@ -7,6 +7,13 @@ WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --timeout 120 --retries 10 -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt
|
||||
|
||||
ARG APP_GIT_SHA=unknown
|
||||
ARG APP_BUILD_TIME=unknown
|
||||
ENV APP_GIT_SHA=${APP_GIT_SHA} \
|
||||
APP_BUILD_TIME=${APP_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${APP_GIT_SHA} \
|
||||
org.opencontainers.image.created=${APP_BUILD_TIME}
|
||||
|
||||
COPY . .
|
||||
|
||||
# 后端使用 SQLite(avatar.db 落在 /app 内),平铺结构以 `uvicorn main:app` 启动
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
import os
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from database import init_db, SessionLocal
|
||||
from database import engine, init_db, SessionLocal
|
||||
from models import Avatar, Authorization, Organization, TokenAccount, TokenPlan, User
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
import routers.avatars
|
||||
@@ -54,7 +55,31 @@ app.mount("/api/files", StaticFiles(directory=UPLOAD_DIR), name="knowledge-files
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return ok({"status": "ok"})
|
||||
checks = _runtime_checks()
|
||||
return ok({
|
||||
"status": "ok" if all(checks.values()) else "degraded",
|
||||
"gitSha": os.getenv("APP_GIT_SHA", "unknown"),
|
||||
"buildTime": os.getenv("APP_BUILD_TIME", "unknown"),
|
||||
"checks": checks,
|
||||
})
|
||||
|
||||
|
||||
def _runtime_checks():
|
||||
return {
|
||||
"database": _database_is_ready(),
|
||||
"uploads": os.path.isdir(UPLOAD_DIR) and os.access(UPLOAD_DIR, os.W_OK),
|
||||
"pdfOcr": importlib.util.find_spec("pymupdf") is not None,
|
||||
}
|
||||
|
||||
|
||||
def _database_is_ready():
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.exec_driver_sql("SELECT 1")
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Database readiness check failed")
|
||||
return False
|
||||
|
||||
|
||||
def seed():
|
||||
|
||||
@@ -5,6 +5,7 @@ pydantic
|
||||
python-multipart
|
||||
httpx
|
||||
pypdf
|
||||
PyMuPDF>=1.24,<2
|
||||
python-docx
|
||||
openpyxl
|
||||
apscheduler>=3.10
|
||||
|
||||
@@ -8,7 +8,8 @@ import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from database import SessionLocal
|
||||
from models import KnowledgeChunk, KnowledgeDoc
|
||||
from models import Avatar, KnowledgeChunk, KnowledgeDoc
|
||||
from services.pdf_ocr_service import extract_scanned_pdf_text
|
||||
import embeddings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -76,7 +77,23 @@ class KnowledgeVectorizer:
|
||||
|
||||
self._set_progress(db, doc, "extracting", 8)
|
||||
text = embeddings.extract_text(path, f".{doc.file_type}")
|
||||
self._set_progress(db, doc, "chunking", 22)
|
||||
if doc.file_type == "pdf" and not text.strip():
|
||||
avatar = db.get(Avatar, doc.avatar_id)
|
||||
if not avatar:
|
||||
raise ValueError("文档所属分身不存在")
|
||||
|
||||
def ocr_progress(done: int, total: int):
|
||||
percent = 8 + int((done / max(1, total)) * 20)
|
||||
self._set_progress(db, doc, "ocr", min(percent, 28))
|
||||
|
||||
self._set_progress(db, doc, "ocr", 8)
|
||||
text = extract_scanned_pdf_text(
|
||||
db,
|
||||
avatar,
|
||||
path,
|
||||
on_progress=ocr_progress,
|
||||
)
|
||||
self._set_progress(db, doc, "chunking", 29)
|
||||
chunks = embeddings.chunk_text(text)
|
||||
if not chunks:
|
||||
raise ValueError("文档没有可建立索引的文字内容")
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""OCR fallback for image-only PDF knowledge documents."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models import Avatar
|
||||
from services.chat_model_config import get_chat_model_config
|
||||
from services.token_billing import (
|
||||
estimate_fallback_usage,
|
||||
release_reservation,
|
||||
reserve_avatar_tokens,
|
||||
settle_reservation,
|
||||
)
|
||||
from services.vision_service import call_vision_model, prepare_image
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PDF_OCR_PROMPT = (
|
||||
"请逐字转录这一页扫描文档中的全部可见文字和表格,只输出转录内容,不要解释,不要使用 Markdown 代码块。"
|
||||
"保留标题、段落、项目编号、数值和自然换行;看不清的内容写作[无法辨认],不要猜测、纠错或补全。"
|
||||
)
|
||||
|
||||
|
||||
def _positive_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
value = default
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def extract_scanned_pdf_text(
|
||||
db: Session,
|
||||
avatar: Avatar,
|
||||
path: str,
|
||||
*,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> str:
|
||||
"""Render and OCR an image-only PDF while preserving page order."""
|
||||
try:
|
||||
import pymupdf
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("扫描型 PDF 识别组件未安装") from exc
|
||||
|
||||
max_pages = _positive_int("KNOWLEDGE_PDF_OCR_MAX_PAGES", 80, 1, 300)
|
||||
render_dpi = _positive_int("KNOWLEDGE_PDF_OCR_DPI", 144, 96, 200)
|
||||
max_attempts = _positive_int("KNOWLEDGE_PDF_OCR_ATTEMPTS", 3, 1, 5)
|
||||
model_config = get_chat_model_config()
|
||||
model = model_config.ocr_model or model_config.vision_model
|
||||
if not model_config.api_key or not model:
|
||||
raise RuntimeError("扫描型 PDF 需要配置视觉 OCR 模型")
|
||||
|
||||
texts: list[str] = []
|
||||
with pymupdf.open(path) as document:
|
||||
total_pages = document.page_count
|
||||
if total_pages <= 0:
|
||||
raise ValueError("PDF 没有可识别页面")
|
||||
if total_pages > max_pages:
|
||||
raise ValueError(
|
||||
f"扫描型 PDF 共 {total_pages} 页,超过单次 OCR 上限 {max_pages} 页,请拆分后上传"
|
||||
)
|
||||
|
||||
scale = render_dpi / 72
|
||||
for page_index in range(total_pages):
|
||||
page = document.load_page(page_index)
|
||||
pixmap = page.get_pixmap(
|
||||
matrix=pymupdf.Matrix(scale, scale),
|
||||
colorspace=pymupdf.csRGB,
|
||||
alpha=False,
|
||||
)
|
||||
prepared = prepare_image(pixmap.tobytes("jpeg", jpg_quality=88))
|
||||
estimate_messages = [{
|
||||
"role": "user",
|
||||
"content": f"[扫描 PDF 第 {page_index + 1}/{total_pages} 页]\n{PDF_OCR_PROMPT}",
|
||||
}]
|
||||
reservation = reserve_avatar_tokens(
|
||||
db,
|
||||
avatar,
|
||||
"knowledge_pdf_ocr",
|
||||
model,
|
||||
estimate_messages,
|
||||
model_config.vision_max_tokens,
|
||||
)
|
||||
try:
|
||||
result = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
result = call_vision_model(
|
||||
prepared,
|
||||
model_config,
|
||||
model=model,
|
||||
prompt=PDF_OCR_PROMPT,
|
||||
json_output=False,
|
||||
)
|
||||
break
|
||||
except RuntimeError:
|
||||
if attempt == max_attempts:
|
||||
raise
|
||||
time.sleep(min(4, attempt))
|
||||
content = str((result or {}).get("content") or "").strip()
|
||||
if not content:
|
||||
raise RuntimeError("扫描型 PDF 页面识别结果为空")
|
||||
settle_reservation(
|
||||
db,
|
||||
reservation,
|
||||
(result or {}).get("usage"),
|
||||
fallback_total=estimate_fallback_usage(estimate_messages, content),
|
||||
)
|
||||
except Exception as exc:
|
||||
release_reservation(db, reservation, str(exc))
|
||||
raise RuntimeError(
|
||||
f"扫描型 PDF 第 {page_index + 1}/{total_pages} 页识别失败:{exc}"
|
||||
) from exc
|
||||
|
||||
texts.append(f"[第 {page_index + 1} 页]\n{content}")
|
||||
if on_progress:
|
||||
on_progress(page_index + 1, total_pages)
|
||||
logger.info(
|
||||
"Scanned PDF OCR completed avatar=%s page=%s/%s",
|
||||
avatar.id,
|
||||
page_index + 1,
|
||||
total_pages,
|
||||
)
|
||||
|
||||
return "\n\n".join(texts).strip()
|
||||
@@ -0,0 +1,33 @@
|
||||
import main
|
||||
|
||||
|
||||
def test_health_reports_release_and_runtime_capabilities(monkeypatch):
|
||||
monkeypatch.setenv("APP_GIT_SHA", "test-sha")
|
||||
monkeypatch.setenv("APP_BUILD_TIME", "2026-09-09T00:00:00Z")
|
||||
monkeypatch.setattr(main, "_runtime_checks", lambda: {
|
||||
"database": True,
|
||||
"uploads": True,
|
||||
"pdfOcr": True,
|
||||
})
|
||||
|
||||
response = main.health()
|
||||
|
||||
assert response["code"] == 200
|
||||
assert response["data"]["status"] == "ok"
|
||||
assert response["data"]["gitSha"] == "test-sha"
|
||||
assert response["data"]["buildTime"] == "2026-09-09T00:00:00Z"
|
||||
assert response["data"]["checks"] == {
|
||||
"database": True,
|
||||
"uploads": True,
|
||||
"pdfOcr": True,
|
||||
}
|
||||
|
||||
|
||||
def test_health_is_degraded_when_a_required_capability_is_missing(monkeypatch):
|
||||
monkeypatch.setattr(main, "_runtime_checks", lambda: {
|
||||
"database": True,
|
||||
"uploads": True,
|
||||
"pdfOcr": False,
|
||||
})
|
||||
|
||||
assert main.health()["data"]["status"] == "degraded"
|
||||
@@ -238,6 +238,53 @@ def test_background_vectorizer_keeps_failure_reason_for_retry(
|
||||
db.close()
|
||||
|
||||
|
||||
def test_background_vectorizer_uses_ocr_for_image_only_pdf(
|
||||
tmp_path: Path,
|
||||
authorization_context,
|
||||
):
|
||||
context = authorization_context
|
||||
with (
|
||||
patch("routers.knowledge.UPLOAD_DIR", str(tmp_path)),
|
||||
patch("routers.knowledge.knowledge_vectorizer.enqueue"),
|
||||
):
|
||||
response = client.post(
|
||||
f"/api/avatar/{context['avatar'].id}/knowledge/docs",
|
||||
headers=context["owner_headers"],
|
||||
files={"file": ("scanned.pdf", b"image-only-pdf", "application/pdf")},
|
||||
)
|
||||
|
||||
payload = response.json()["data"]
|
||||
progress = []
|
||||
with (
|
||||
patch("services.knowledge_vectorizer.UPLOAD_DIR", str(tmp_path)),
|
||||
patch("services.knowledge_vectorizer.embeddings.extract_text", return_value=""),
|
||||
patch(
|
||||
"services.knowledge_vectorizer.extract_scanned_pdf_text",
|
||||
side_effect=lambda _db, _avatar, _path, on_progress: (
|
||||
on_progress(1, 2), on_progress(2, 2), "扫描页文字"
|
||||
)[-1],
|
||||
) as ocr,
|
||||
patch("services.knowledge_vectorizer.embeddings.embed", return_value=[[1.0, 0.0]]),
|
||||
patch.object(knowledge_vectorizer, "_set_progress", wraps=knowledge_vectorizer._set_progress) as set_progress,
|
||||
):
|
||||
knowledge_vectorizer.vectorize_document(payload["id"])
|
||||
progress = [(call.args[2], call.args[3]) for call in set_progress.call_args_list]
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
stored = db.query(KnowledgeDoc).filter(KnowledgeDoc.id == payload["id"]).one()
|
||||
assert stored.status == "ready"
|
||||
assert stored.chunk_count == 1
|
||||
assert ("ocr", 18) in progress
|
||||
assert ("ocr", 28) in progress
|
||||
ocr.assert_called_once()
|
||||
db.query(KnowledgeChunk).filter(KnowledgeChunk.doc_id == stored.id).delete()
|
||||
db.delete(stored)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_retry_queues_a_failed_document_again(
|
||||
tmp_path: Path,
|
||||
authorization_context,
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from services.pdf_ocr_service import extract_scanned_pdf_text
|
||||
|
||||
|
||||
class FakePixmap:
|
||||
def tobytes(self, *_args, **_kwargs):
|
||||
return b"jpeg-page"
|
||||
|
||||
|
||||
class FakePage:
|
||||
def get_pixmap(self, **_kwargs):
|
||||
return FakePixmap()
|
||||
|
||||
|
||||
class FakeDocument:
|
||||
page_count = 2
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def load_page(self, _index):
|
||||
return FakePage()
|
||||
|
||||
|
||||
def test_scanned_pdf_ocr_preserves_page_order_and_reports_progress(monkeypatch):
|
||||
fake_pymupdf = SimpleNamespace(
|
||||
open=lambda _path: FakeDocument(),
|
||||
Matrix=lambda x, y: (x, y),
|
||||
csRGB="rgb",
|
||||
)
|
||||
monkeypatch.setitem(__import__("sys").modules, "pymupdf", fake_pymupdf)
|
||||
progress = []
|
||||
reservation = SimpleNamespace()
|
||||
config = SimpleNamespace(
|
||||
api_key="configured",
|
||||
ocr_model="qwen-vl-ocr",
|
||||
vision_model="vision",
|
||||
vision_max_tokens=2048,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("services.pdf_ocr_service.get_chat_model_config", return_value=config),
|
||||
patch("services.pdf_ocr_service.prepare_image", return_value=SimpleNamespace()),
|
||||
patch(
|
||||
"services.pdf_ocr_service.call_vision_model",
|
||||
side_effect=[
|
||||
{"content": "第一页文字", "usage": {"total_tokens": 10}},
|
||||
{"content": "第二页文字", "usage": {"total_tokens": 12}},
|
||||
],
|
||||
),
|
||||
patch("services.pdf_ocr_service.reserve_avatar_tokens", return_value=reservation) as reserve,
|
||||
patch("services.pdf_ocr_service.settle_reservation") as settle,
|
||||
):
|
||||
text = extract_scanned_pdf_text(
|
||||
MagicMock(),
|
||||
SimpleNamespace(id="avatar-1"),
|
||||
"/tmp/scanned.pdf",
|
||||
on_progress=lambda done, total: progress.append((done, total)),
|
||||
)
|
||||
|
||||
assert text == "[第 1 页]\n第一页文字\n\n[第 2 页]\n第二页文字"
|
||||
assert progress == [(1, 2), (2, 2)]
|
||||
assert reserve.call_count == 2
|
||||
assert settle.call_count == 2
|
||||
|
||||
|
||||
def test_scanned_pdf_ocr_releases_tokens_after_retries_fail(monkeypatch):
|
||||
fake_document = FakeDocument()
|
||||
fake_document.page_count = 1
|
||||
fake_pymupdf = SimpleNamespace(
|
||||
open=lambda _path: fake_document,
|
||||
Matrix=lambda x, y: (x, y),
|
||||
csRGB="rgb",
|
||||
)
|
||||
monkeypatch.setitem(__import__("sys").modules, "pymupdf", fake_pymupdf)
|
||||
monkeypatch.setenv("KNOWLEDGE_PDF_OCR_ATTEMPTS", "2")
|
||||
reservation = SimpleNamespace()
|
||||
config = SimpleNamespace(
|
||||
api_key="configured",
|
||||
ocr_model="qwen-vl-ocr",
|
||||
vision_model="vision",
|
||||
vision_max_tokens=2048,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("services.pdf_ocr_service.get_chat_model_config", return_value=config),
|
||||
patch("services.pdf_ocr_service.prepare_image", return_value=SimpleNamespace()),
|
||||
patch("services.pdf_ocr_service.call_vision_model", side_effect=RuntimeError("timeout")) as call,
|
||||
patch("services.pdf_ocr_service.reserve_avatar_tokens", return_value=reservation),
|
||||
patch("services.pdf_ocr_service.release_reservation") as release,
|
||||
patch("services.pdf_ocr_service.time.sleep"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="第 1/1 页识别失败"):
|
||||
extract_scanned_pdf_text(MagicMock(), SimpleNamespace(id="avatar-1"), "/tmp/scanned.pdf")
|
||||
|
||||
assert call.call_count == 2
|
||||
release.assert_called_once()
|
||||
@@ -1,42 +1,56 @@
|
||||
# 会会数字分身 —— Docker 测试实例(独立端口,不干扰现有 :8088 huihui 部署)
|
||||
services:
|
||||
avatar-backend:
|
||||
build: ./backend
|
||||
image: avatar-test-backend:latest
|
||||
build:
|
||||
context: ./backend
|
||||
args:
|
||||
APP_GIT_SHA: ${APP_GIT_SHA:?APP_GIT_SHA must be the full release commit}
|
||||
APP_BUILD_TIME: ${APP_BUILD_TIME:?APP_BUILD_TIME must be set}
|
||||
image: avatar-test-backend:${APP_GIT_SHA}
|
||||
container_name: avatar-test-backend
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
DATABASE_URL: sqlite:////data/avatar.db
|
||||
DATABASE_URL: sqlite:////data/db/avatar.db
|
||||
UPLOAD_DIR: /data/uploads
|
||||
CHAT_MODEL_CONFIG_URL: http://host.docker.internal:8000/api/ai-models/runtime/digital-avatar
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- avatar-data:/data
|
||||
# Mount the directory, not only avatar.db: SQLite WAL/SHM files must survive recreation.
|
||||
- ${AVATAR_DB_DIR:?AVATAR_DB_DIR must contain the persistent avatar.db}:/data/db
|
||||
- ${AVATAR_UPLOAD_DIR:?AVATAR_UPLOAD_DIR must point to persistent uploads}:/data/uploads
|
||||
expose:
|
||||
- "8000"
|
||||
ports:
|
||||
- "8011:8000" # 仅用于直接调试 API;前端经内部网络访问,不走 host 端口
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import json,urllib.request; d=json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=5))['data']; assert d['status']=='ok' and all(d['checks'].values())"]
|
||||
interval: 10s
|
||||
timeout: 8s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
networks:
|
||||
- avatar-net
|
||||
|
||||
avatar-frontend:
|
||||
build: .
|
||||
image: avatar-test-frontend:latest
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_GIT_SHA: ${APP_GIT_SHA:?APP_GIT_SHA must be the full release commit}
|
||||
APP_BUILD_TIME: ${APP_BUILD_TIME:?APP_BUILD_TIME must be set}
|
||||
image: avatar-test-frontend:${APP_GIT_SHA}
|
||||
container_name: avatar-test-frontend
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8099:80" # 浏览器访问 http://<host>:8099
|
||||
depends_on:
|
||||
- avatar-backend
|
||||
avatar-backend:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- avatar-net
|
||||
|
||||
networks:
|
||||
avatar-net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
avatar-data:
|
||||
|
||||
@@ -61,7 +61,9 @@ WECHAT_VIRTUAL_PRODUCT_2=<100元套餐商品ID>
|
||||
WECHAT_VIRTUAL_PRODUCT_3=<1000元套餐商品ID>
|
||||
WECHAT_VIRTUAL_PRODUCT_4=<10000元套餐商品ID>
|
||||
|
||||
DATABASE_URL=sqlite:////data/avatar.db
|
||||
AVATAR_DB_DIR=/srv/digital-avatar/data/db
|
||||
AVATAR_UPLOAD_DIR=/srv/digital-avatar/data/uploads
|
||||
DATABASE_URL=sqlite:////data/db/avatar.db
|
||||
UPLOAD_DIR=/data/uploads
|
||||
CHAT_MODEL_CONFIG_URL=http://<huihuisquare-api>/api/ai-models/runtime/digital-avatar
|
||||
EMBEDDING_API_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
@@ -74,13 +76,17 @@ VISION_OCR_MODEL=qwen-vl-ocr
|
||||
VISION_MAX_OUTPUT_TOKENS=2048
|
||||
VISION_TIMEOUT_SECONDS=90
|
||||
VISION_TOKEN_RESERVE=12000
|
||||
APP_GIT_SHA=<本次发布的完整提交SHA>
|
||||
APP_BUILD_TIME=<UTC ISO-8601构建时间>
|
||||
CHAT_IMAGE_MAX_BYTES=8388608
|
||||
CHAT_IMAGE_MAX_PIXELS=16000000
|
||||
CHAT_ATTACHMENT_RETENTION_HOURS=24
|
||||
CHAT_ATTACHMENT_CLEANUP_MINUTES=60
|
||||
```
|
||||
|
||||
如生产 AI 配置中心不可用,还应提供当前项目支持的 `OPENAI_API_KEY`、`OPENAI_BASE_URL`、`CHAT_MODEL` 等兜底配置。`/data` 必须挂载持久卷,数据库与知识库文件不可存放在容器临时层。
|
||||
如生产 AI 配置中心不可用,还应提供当前项目支持的 `OPENAI_API_KEY`、`OPENAI_BASE_URL`、`CHAT_MODEL` 等兜底配置。数据库文件与上传目录必须从宿主机显式挂载,不能存放在容器临时层。
|
||||
|
||||
`AVATAR_DB_DIR` 和 `AVATAR_UPLOAD_DIR` 必须是已备份的宿主机绝对路径,编排缺少任一变量都会直接拒绝构建或启动,防止误挂空卷造成用户、分身或知识库“丢失”的假象。SQLite 必须挂载整个数据库目录,不能只挂载 `avatar.db` 单文件,否则 `avatar.db-wal` 和 `avatar.db-shm` 会留在容器临时层,换容器后可能出现数据状态回退。
|
||||
|
||||
`EMBEDDING_API_URL` 同时支持 OpenAI 兼容基础地址(如上面的 `/v1`)和完整的 `/v1/embeddings` 地址,后端会统一请求 `/embeddings`。发布后必须在后端容器内执行一次最小向量探针,确认返回向量数量和维度,而不能只检查 `/api/health`。
|
||||
|
||||
@@ -97,7 +103,7 @@ App 与 H5 积分充值使用会会支付体系的 `payment-v3/payment/pay`,
|
||||
```bash
|
||||
BACKUP_DIR="backups/$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
cp /srv/digital-avatar/data/avatar.db "$BACKUP_DIR/"
|
||||
cp /srv/digital-avatar/data/db/avatar.db "$BACKUP_DIR/"
|
||||
tar -C /srv/digital-avatar/data -czf "$BACKUP_DIR/uploads.tgz" uploads
|
||||
```
|
||||
|
||||
@@ -107,13 +113,22 @@ tar -C /srv/digital-avatar/data -czf "$BACKUP_DIR/uploads.tgz" uploads
|
||||
git fetch origin
|
||||
git checkout <已验收的提交SHA>
|
||||
cd digital-avatar-app
|
||||
docker compose build --pull avatar-backend avatar-frontend
|
||||
docker compose up -d avatar-backend avatar-frontend
|
||||
export APP_GIT_SHA="$(git rev-parse HEAD)"
|
||||
export APP_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
docker compose build --pull --no-cache avatar-backend avatar-frontend
|
||||
docker compose up -d --force-recreate --wait avatar-backend avatar-frontend
|
||||
docker compose ps
|
||||
curl -fsS http://127.0.0.1:8099/api/health
|
||||
python3 scripts/verify-deployment.py \
|
||||
https://digital.99hui.com "$APP_GIT_SHA" \
|
||||
--backend-container avatar-backend \
|
||||
--frontend-container avatar-frontend \
|
||||
--expected-db-source /srv/digital-avatar/data/db \
|
||||
--expected-upload-source /srv/digital-avatar/data/uploads
|
||||
docker compose exec avatar-backend python -c 'import embeddings; v=embeddings.embed(["部署向量探针"]); print(len(v), len(v[0]))'
|
||||
```
|
||||
|
||||
Jenkins 必须以 `verify-deployment.py` 返回成功作为发布成功条件,不能只以镜像构建或容器启动成功作为条件。脚本会同时核对公网前后端 Git SHA、数据库可读、上传目录可写、PDF OCR 依赖和宿主机数据挂载;任意一项不一致都会返回非零状态并阻止发布标绿。镜像使用 Git SHA 标签,不再依赖可被旧缓存覆盖的 `latest`。
|
||||
|
||||
生产编排应把示例中的测试端口改为内网暴露,由统一 HTTPS 网关接入。后端暂时使用 SQLite,必须保持单实例写入;若扩展为多后端实例,应先迁移到 PostgreSQL,并把延迟接管任务改为共享队列。
|
||||
|
||||
## 4. 网关要求
|
||||
@@ -153,7 +168,7 @@ location /api/ {
|
||||
5. 使用过期或伪造 token 时进入登录页并显示凭证失效,不得继续访问旧用户数据。
|
||||
6. 分身聊天 SSE 逐段输出正常,Markdown 正常渲染,知识库优先级和积分扣费正常。
|
||||
7. 开启 BOXIM 主动接管后保持在线,默认三分钟回复、自定义等待时间、已读回执、分身防回环和主人发言暂停均正常。
|
||||
8. 重建容器后数据库、头像、知识库文档仍存在,`/api/health` 返回成功。
|
||||
8. 重建容器后数据库、头像、知识库文档仍存在,`/api/health` 的 `gitSha` 与发布 SHA 一致,`database`、`uploads`、`pdfOcr` 三项检查均为 `true`。
|
||||
9. `https://digital.99hui.com/api/health` 可访问,证书域名和有效期正确,HTTP 自动跳转 HTTPS。
|
||||
10. 微信和支付宝各创建一笔最小套餐订单,未付款时积分不变;支付成功后回调到账一次,重复回调积分不重复增加。
|
||||
11. 微信虚拟支付在沙箱环境完成下单、支付回调、查单兜底和退款回调;错误 OpenID、商品、环境或金额均被拒绝。
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail a deployment unless frontend and backend run the expected release."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
with urllib.request.urlopen(url, timeout=20) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"{url} returned HTTP {response.status}")
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("base_url", help="Public site URL, for example https://digital.99hui.com")
|
||||
parser.add_argument("expected_sha", help="Full Git commit SHA being deployed")
|
||||
parser.add_argument("--backend-container", help="Backend container name for image and mount checks")
|
||||
parser.add_argument("--frontend-container", help="Frontend container name for image checks")
|
||||
parser.add_argument("--expected-db-source", help="Required host directory mounted for SQLite and its WAL files")
|
||||
parser.add_argument("--expected-upload-source", help="Required host source mounted as the upload directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
errors = []
|
||||
try:
|
||||
health = fetch_json(f"{base_url}/api/health").get("data") or {}
|
||||
except Exception as exc:
|
||||
errors.append(f"cannot read backend release metadata: {exc}")
|
||||
health = {}
|
||||
try:
|
||||
frontend = fetch_json(f"{base_url}/version.json")
|
||||
except Exception as exc:
|
||||
errors.append(f"cannot read frontend release metadata: {exc}")
|
||||
frontend = {}
|
||||
|
||||
if health.get("status") != "ok":
|
||||
errors.append(f"backend status is {health.get('status')!r}")
|
||||
failed_checks = [name for name, passed in (health.get("checks") or {}).items() if not passed]
|
||||
if failed_checks:
|
||||
errors.append("backend checks failed: " + ", ".join(failed_checks))
|
||||
if health.get("gitSha") != args.expected_sha:
|
||||
errors.append(f"backend SHA is {health.get('gitSha')!r}")
|
||||
if frontend.get("gitSha") != args.expected_sha:
|
||||
errors.append(f"frontend SHA is {frontend.get('gitSha')!r}")
|
||||
|
||||
if args.backend_container:
|
||||
backend = inspect_container(args.backend_container, errors)
|
||||
check_container_revision(backend, args.expected_sha, "backend", errors)
|
||||
check_mount(backend, args.expected_db_source, "database", errors)
|
||||
check_mount(backend, args.expected_upload_source, "uploads", errors)
|
||||
elif args.expected_db_source or args.expected_upload_source:
|
||||
errors.append("--backend-container is required when checking data mounts")
|
||||
|
||||
if args.frontend_container:
|
||||
frontend_container = inspect_container(args.frontend_container, errors)
|
||||
check_container_revision(frontend_container, args.expected_sha, "frontend", errors)
|
||||
|
||||
if errors:
|
||||
print("Deployment verification failed:", file=sys.stderr)
|
||||
for error in errors:
|
||||
print(f"- {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"Deployment verified: {args.expected_sha}")
|
||||
print("Backend checks: database, uploads, pdfOcr")
|
||||
return 0
|
||||
|
||||
|
||||
def inspect_container(name, errors):
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
["docker", "inspect", name], universal_newlines=True
|
||||
)
|
||||
return json.loads(output)[0]
|
||||
except Exception as exc:
|
||||
errors.append(f"cannot inspect container {name!r}: {exc}")
|
||||
return {}
|
||||
|
||||
|
||||
def check_container_revision(container, expected_sha, label, errors):
|
||||
actual = ((container.get("Config") or {}).get("Labels") or {}).get(
|
||||
"org.opencontainers.image.revision"
|
||||
)
|
||||
if actual != expected_sha:
|
||||
errors.append(f"{label} container image SHA is {actual!r}")
|
||||
|
||||
|
||||
def check_mount(container, expected_source, label, errors):
|
||||
if not expected_source:
|
||||
return
|
||||
expected = os.path.realpath(expected_source)
|
||||
sources = {
|
||||
os.path.realpath(mount.get("Source", ""))
|
||||
for mount in container.get("Mounts") or []
|
||||
}
|
||||
if expected not in sources:
|
||||
errors.append(f"{label} mount source {expected!r} is not attached")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -179,7 +179,7 @@ const permissionItems: Array<{
|
||||
},
|
||||
{
|
||||
key: 'publish',
|
||||
title: '发布微博内容',
|
||||
title: '发布微播内容',
|
||||
description: '允许分身自动发布动态内容',
|
||||
tone: 'green',
|
||||
},
|
||||
@@ -691,7 +691,9 @@ svg {
|
||||
font-weight: 400;
|
||||
line-height: 1.45;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
white-space: normal;
|
||||
overflow-wrap: anywhere;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.permission-row.takeover .permission-copy small {
|
||||
|
||||
@@ -147,7 +147,7 @@ const documentState = (doc: any) => {
|
||||
if (['uploaded', 'parsing'].includes(String(doc.status || '').toLowerCase())) {
|
||||
const stage = String(doc.indexStage || 'queued').toLowerCase()
|
||||
const labels: Record<string, string> = {
|
||||
queued: '等待处理', extracting: '解析文档', chunking: '切分文本', embedding: '向量化中'
|
||||
queued: '等待处理', extracting: '解析文档', ocr: '扫描件识别', chunking: '切分文本', embedding: '向量化中'
|
||||
}
|
||||
const progress = Math.max(0, Math.min(99, Number(doc.indexProgress || 0)))
|
||||
return { tone: 'pending', label: labels[stage] || '处理中', detail: `${labels[stage] || '正在建立知识索引'} ${progress}%`, progress }
|
||||
|
||||
Reference in New Issue
Block a user