Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7e7aa06903 |
@@ -1,16 +1,12 @@
|
|||||||
# 构建阶段:安装依赖并打包 H5
|
# 构建阶段:安装依赖并打包 H5
|
||||||
FROM node:18-alpine AS build
|
FROM node:18-alpine AS build
|
||||||
|
|
||||||
ARG APP_GIT_SHA=unknown
|
|
||||||
ARG APP_BUILD_TIME=unknown
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY package*.json ./
|
COPY package*.json ./
|
||||||
RUN npm ci
|
RUN npm ci
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN printf '{"gitSha":"%s","buildTime":"%s"}\n' "$APP_GIT_SHA" "$APP_BUILD_TIME" > public/version.json
|
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|
||||||
# 运行阶段:nginx 托管静态资源并反向代理 /api 到后端
|
# 运行阶段:nginx 托管静态资源并反向代理 /api 到后端
|
||||||
@@ -18,11 +14,6 @@ RUN npm run build
|
|||||||
# 新版 nginx(>=1.31) 用 pwrite 写 pid 文件会被拦导致致命退出;1.28 用 write() 可正常启动。
|
# 新版 nginx(>=1.31) 用 pwrite 写 pid 文件会被拦导致致命退出;1.28 用 write() 可正常启动。
|
||||||
FROM nginx:1.28-alpine
|
FROM nginx:1.28-alpine
|
||||||
|
|
||||||
ARG APP_GIT_SHA=unknown
|
|
||||||
ARG APP_BUILD_TIME=unknown
|
|
||||||
LABEL org.opencontainers.image.revision=${APP_GIT_SHA} \
|
|
||||||
org.opencontainers.image.created=${APP_BUILD_TIME}
|
|
||||||
|
|
||||||
COPY --from=build /app/dist /usr/share/nginx/html
|
COPY --from=build /app/dist /usr/share/nginx/html
|
||||||
# 覆盖 nginx 默认主配置(含唯一可写的 pid /tmp/nginx.pid,规避受限容器内 /run 不可写导致反复重启)
|
# 覆盖 nginx 默认主配置(含唯一可写的 pid /tmp/nginx.pid,规避受限容器内 /run 不可写导致反复重启)
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf
|
COPY nginx.conf /etc/nginx/nginx.conf
|
||||||
|
|||||||
@@ -7,13 +7,6 @@ WORKDIR /app
|
|||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir --timeout 120 --retries 10 -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt
|
RUN pip install --no-cache-dir --timeout 120 --retries 10 -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt
|
||||||
|
|
||||||
ARG APP_GIT_SHA=unknown
|
|
||||||
ARG APP_BUILD_TIME=unknown
|
|
||||||
ENV APP_GIT_SHA=${APP_GIT_SHA} \
|
|
||||||
APP_BUILD_TIME=${APP_BUILD_TIME}
|
|
||||||
LABEL org.opencontainers.image.revision=${APP_GIT_SHA} \
|
|
||||||
org.opencontainers.image.created=${APP_BUILD_TIME}
|
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# 后端使用 SQLite(avatar.db 落在 /app 内),平铺结构以 `uvicorn main:app` 启动
|
# 后端使用 SQLite(avatar.db 落在 /app 内),平铺结构以 `uvicorn main:app` 启动
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
from apscheduler.triggers.interval import IntervalTrigger
|
from apscheduler.triggers.interval import IntervalTrigger
|
||||||
|
|
||||||
from database import engine, init_db, SessionLocal
|
from database import init_db, SessionLocal
|
||||||
from models import Avatar, Authorization, Organization, TokenAccount, TokenPlan, User
|
from models import Avatar, Authorization, Organization, TokenAccount, TokenPlan, User
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
import routers.avatars
|
import routers.avatars
|
||||||
@@ -55,31 +54,7 @@ app.mount("/api/files", StaticFiles(directory=UPLOAD_DIR), name="knowledge-files
|
|||||||
|
|
||||||
@app.get("/api/health")
|
@app.get("/api/health")
|
||||||
def health():
|
def health():
|
||||||
checks = _runtime_checks()
|
return ok({"status": "ok"})
|
||||||
return ok({
|
|
||||||
"status": "ok" if all(checks.values()) else "degraded",
|
|
||||||
"gitSha": os.getenv("APP_GIT_SHA", "unknown"),
|
|
||||||
"buildTime": os.getenv("APP_BUILD_TIME", "unknown"),
|
|
||||||
"checks": checks,
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _runtime_checks():
|
|
||||||
return {
|
|
||||||
"database": _database_is_ready(),
|
|
||||||
"uploads": os.path.isdir(UPLOAD_DIR) and os.access(UPLOAD_DIR, os.W_OK),
|
|
||||||
"pdfOcr": importlib.util.find_spec("pymupdf") is not None,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _database_is_ready():
|
|
||||||
try:
|
|
||||||
with engine.connect() as connection:
|
|
||||||
connection.exec_driver_sql("SELECT 1")
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
logger.exception("Database readiness check failed")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def seed():
|
def seed():
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ pydantic
|
|||||||
python-multipart
|
python-multipart
|
||||||
httpx
|
httpx
|
||||||
pypdf
|
pypdf
|
||||||
PyMuPDF>=1.24,<2
|
|
||||||
python-docx
|
python-docx
|
||||||
openpyxl
|
openpyxl
|
||||||
apscheduler>=3.10
|
apscheduler>=3.10
|
||||||
|
|||||||
@@ -8,8 +8,7 @@ import threading
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from database import SessionLocal
|
from database import SessionLocal
|
||||||
from models import Avatar, KnowledgeChunk, KnowledgeDoc
|
from models import KnowledgeChunk, KnowledgeDoc
|
||||||
from services.pdf_ocr_service import extract_scanned_pdf_text
|
|
||||||
import embeddings
|
import embeddings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -77,23 +76,7 @@ class KnowledgeVectorizer:
|
|||||||
|
|
||||||
self._set_progress(db, doc, "extracting", 8)
|
self._set_progress(db, doc, "extracting", 8)
|
||||||
text = embeddings.extract_text(path, f".{doc.file_type}")
|
text = embeddings.extract_text(path, f".{doc.file_type}")
|
||||||
if doc.file_type == "pdf" and not text.strip():
|
self._set_progress(db, doc, "chunking", 22)
|
||||||
avatar = db.get(Avatar, doc.avatar_id)
|
|
||||||
if not avatar:
|
|
||||||
raise ValueError("文档所属分身不存在")
|
|
||||||
|
|
||||||
def ocr_progress(done: int, total: int):
|
|
||||||
percent = 8 + int((done / max(1, total)) * 20)
|
|
||||||
self._set_progress(db, doc, "ocr", min(percent, 28))
|
|
||||||
|
|
||||||
self._set_progress(db, doc, "ocr", 8)
|
|
||||||
text = extract_scanned_pdf_text(
|
|
||||||
db,
|
|
||||||
avatar,
|
|
||||||
path,
|
|
||||||
on_progress=ocr_progress,
|
|
||||||
)
|
|
||||||
self._set_progress(db, doc, "chunking", 29)
|
|
||||||
chunks = embeddings.chunk_text(text)
|
chunks = embeddings.chunk_text(text)
|
||||||
if not chunks:
|
if not chunks:
|
||||||
raise ValueError("文档没有可建立索引的文字内容")
|
raise ValueError("文档没有可建立索引的文字内容")
|
||||||
|
|||||||
@@ -1,130 +0,0 @@
|
|||||||
"""OCR fallback for image-only PDF knowledge documents."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
from typing import Callable
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from models import Avatar
|
|
||||||
from services.chat_model_config import get_chat_model_config
|
|
||||||
from services.token_billing import (
|
|
||||||
estimate_fallback_usage,
|
|
||||||
release_reservation,
|
|
||||||
reserve_avatar_tokens,
|
|
||||||
settle_reservation,
|
|
||||||
)
|
|
||||||
from services.vision_service import call_vision_model, prepare_image
|
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
PDF_OCR_PROMPT = (
|
|
||||||
"请逐字转录这一页扫描文档中的全部可见文字和表格,只输出转录内容,不要解释,不要使用 Markdown 代码块。"
|
|
||||||
"保留标题、段落、项目编号、数值和自然换行;看不清的内容写作[无法辨认],不要猜测、纠错或补全。"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _positive_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
|
||||||
try:
|
|
||||||
value = int(os.getenv(name, str(default)))
|
|
||||||
except ValueError:
|
|
||||||
value = default
|
|
||||||
return max(minimum, min(maximum, value))
|
|
||||||
|
|
||||||
|
|
||||||
def extract_scanned_pdf_text(
|
|
||||||
db: Session,
|
|
||||||
avatar: Avatar,
|
|
||||||
path: str,
|
|
||||||
*,
|
|
||||||
on_progress: Callable[[int, int], None] | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render and OCR an image-only PDF while preserving page order."""
|
|
||||||
try:
|
|
||||||
import pymupdf
|
|
||||||
except ImportError as exc:
|
|
||||||
raise RuntimeError("扫描型 PDF 识别组件未安装") from exc
|
|
||||||
|
|
||||||
max_pages = _positive_int("KNOWLEDGE_PDF_OCR_MAX_PAGES", 80, 1, 300)
|
|
||||||
render_dpi = _positive_int("KNOWLEDGE_PDF_OCR_DPI", 144, 96, 200)
|
|
||||||
max_attempts = _positive_int("KNOWLEDGE_PDF_OCR_ATTEMPTS", 3, 1, 5)
|
|
||||||
model_config = get_chat_model_config()
|
|
||||||
model = model_config.ocr_model or model_config.vision_model
|
|
||||||
if not model_config.api_key or not model:
|
|
||||||
raise RuntimeError("扫描型 PDF 需要配置视觉 OCR 模型")
|
|
||||||
|
|
||||||
texts: list[str] = []
|
|
||||||
with pymupdf.open(path) as document:
|
|
||||||
total_pages = document.page_count
|
|
||||||
if total_pages <= 0:
|
|
||||||
raise ValueError("PDF 没有可识别页面")
|
|
||||||
if total_pages > max_pages:
|
|
||||||
raise ValueError(
|
|
||||||
f"扫描型 PDF 共 {total_pages} 页,超过单次 OCR 上限 {max_pages} 页,请拆分后上传"
|
|
||||||
)
|
|
||||||
|
|
||||||
scale = render_dpi / 72
|
|
||||||
for page_index in range(total_pages):
|
|
||||||
page = document.load_page(page_index)
|
|
||||||
pixmap = page.get_pixmap(
|
|
||||||
matrix=pymupdf.Matrix(scale, scale),
|
|
||||||
colorspace=pymupdf.csRGB,
|
|
||||||
alpha=False,
|
|
||||||
)
|
|
||||||
prepared = prepare_image(pixmap.tobytes("jpeg", jpg_quality=88))
|
|
||||||
estimate_messages = [{
|
|
||||||
"role": "user",
|
|
||||||
"content": f"[扫描 PDF 第 {page_index + 1}/{total_pages} 页]\n{PDF_OCR_PROMPT}",
|
|
||||||
}]
|
|
||||||
reservation = reserve_avatar_tokens(
|
|
||||||
db,
|
|
||||||
avatar,
|
|
||||||
"knowledge_pdf_ocr",
|
|
||||||
model,
|
|
||||||
estimate_messages,
|
|
||||||
model_config.vision_max_tokens,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
result = None
|
|
||||||
for attempt in range(1, max_attempts + 1):
|
|
||||||
try:
|
|
||||||
result = call_vision_model(
|
|
||||||
prepared,
|
|
||||||
model_config,
|
|
||||||
model=model,
|
|
||||||
prompt=PDF_OCR_PROMPT,
|
|
||||||
json_output=False,
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except RuntimeError:
|
|
||||||
if attempt == max_attempts:
|
|
||||||
raise
|
|
||||||
time.sleep(min(4, attempt))
|
|
||||||
content = str((result or {}).get("content") or "").strip()
|
|
||||||
if not content:
|
|
||||||
raise RuntimeError("扫描型 PDF 页面识别结果为空")
|
|
||||||
settle_reservation(
|
|
||||||
db,
|
|
||||||
reservation,
|
|
||||||
(result or {}).get("usage"),
|
|
||||||
fallback_total=estimate_fallback_usage(estimate_messages, content),
|
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
release_reservation(db, reservation, str(exc))
|
|
||||||
raise RuntimeError(
|
|
||||||
f"扫描型 PDF 第 {page_index + 1}/{total_pages} 页识别失败:{exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
texts.append(f"[第 {page_index + 1} 页]\n{content}")
|
|
||||||
if on_progress:
|
|
||||||
on_progress(page_index + 1, total_pages)
|
|
||||||
logger.info(
|
|
||||||
"Scanned PDF OCR completed avatar=%s page=%s/%s",
|
|
||||||
avatar.id,
|
|
||||||
page_index + 1,
|
|
||||||
total_pages,
|
|
||||||
)
|
|
||||||
|
|
||||||
return "\n\n".join(texts).strip()
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
import main
|
|
||||||
|
|
||||||
|
|
||||||
def test_health_reports_release_and_runtime_capabilities(monkeypatch):
|
|
||||||
monkeypatch.setenv("APP_GIT_SHA", "test-sha")
|
|
||||||
monkeypatch.setenv("APP_BUILD_TIME", "2026-09-09T00:00:00Z")
|
|
||||||
monkeypatch.setattr(main, "_runtime_checks", lambda: {
|
|
||||||
"database": True,
|
|
||||||
"uploads": True,
|
|
||||||
"pdfOcr": True,
|
|
||||||
})
|
|
||||||
|
|
||||||
response = main.health()
|
|
||||||
|
|
||||||
assert response["code"] == 200
|
|
||||||
assert response["data"]["status"] == "ok"
|
|
||||||
assert response["data"]["gitSha"] == "test-sha"
|
|
||||||
assert response["data"]["buildTime"] == "2026-09-09T00:00:00Z"
|
|
||||||
assert response["data"]["checks"] == {
|
|
||||||
"database": True,
|
|
||||||
"uploads": True,
|
|
||||||
"pdfOcr": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_health_is_degraded_when_a_required_capability_is_missing(monkeypatch):
|
|
||||||
monkeypatch.setattr(main, "_runtime_checks", lambda: {
|
|
||||||
"database": True,
|
|
||||||
"uploads": True,
|
|
||||||
"pdfOcr": False,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert main.health()["data"]["status"] == "degraded"
|
|
||||||
@@ -238,53 +238,6 @@ def test_background_vectorizer_keeps_failure_reason_for_retry(
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def test_background_vectorizer_uses_ocr_for_image_only_pdf(
|
|
||||||
tmp_path: Path,
|
|
||||||
authorization_context,
|
|
||||||
):
|
|
||||||
context = authorization_context
|
|
||||||
with (
|
|
||||||
patch("routers.knowledge.UPLOAD_DIR", str(tmp_path)),
|
|
||||||
patch("routers.knowledge.knowledge_vectorizer.enqueue"),
|
|
||||||
):
|
|
||||||
response = client.post(
|
|
||||||
f"/api/avatar/{context['avatar'].id}/knowledge/docs",
|
|
||||||
headers=context["owner_headers"],
|
|
||||||
files={"file": ("scanned.pdf", b"image-only-pdf", "application/pdf")},
|
|
||||||
)
|
|
||||||
|
|
||||||
payload = response.json()["data"]
|
|
||||||
progress = []
|
|
||||||
with (
|
|
||||||
patch("services.knowledge_vectorizer.UPLOAD_DIR", str(tmp_path)),
|
|
||||||
patch("services.knowledge_vectorizer.embeddings.extract_text", return_value=""),
|
|
||||||
patch(
|
|
||||||
"services.knowledge_vectorizer.extract_scanned_pdf_text",
|
|
||||||
side_effect=lambda _db, _avatar, _path, on_progress: (
|
|
||||||
on_progress(1, 2), on_progress(2, 2), "扫描页文字"
|
|
||||||
)[-1],
|
|
||||||
) as ocr,
|
|
||||||
patch("services.knowledge_vectorizer.embeddings.embed", return_value=[[1.0, 0.0]]),
|
|
||||||
patch.object(knowledge_vectorizer, "_set_progress", wraps=knowledge_vectorizer._set_progress) as set_progress,
|
|
||||||
):
|
|
||||||
knowledge_vectorizer.vectorize_document(payload["id"])
|
|
||||||
progress = [(call.args[2], call.args[3]) for call in set_progress.call_args_list]
|
|
||||||
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
stored = db.query(KnowledgeDoc).filter(KnowledgeDoc.id == payload["id"]).one()
|
|
||||||
assert stored.status == "ready"
|
|
||||||
assert stored.chunk_count == 1
|
|
||||||
assert ("ocr", 18) in progress
|
|
||||||
assert ("ocr", 28) in progress
|
|
||||||
ocr.assert_called_once()
|
|
||||||
db.query(KnowledgeChunk).filter(KnowledgeChunk.doc_id == stored.id).delete()
|
|
||||||
db.delete(stored)
|
|
||||||
db.commit()
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_retry_queues_a_failed_document_again(
|
def test_retry_queues_a_failed_document_again(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
authorization_context,
|
authorization_context,
|
||||||
|
|||||||
@@ -1,104 +0,0 @@
|
|||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from services.pdf_ocr_service import extract_scanned_pdf_text
|
|
||||||
|
|
||||||
|
|
||||||
class FakePixmap:
|
|
||||||
def tobytes(self, *_args, **_kwargs):
|
|
||||||
return b"jpeg-page"
|
|
||||||
|
|
||||||
|
|
||||||
class FakePage:
|
|
||||||
def get_pixmap(self, **_kwargs):
|
|
||||||
return FakePixmap()
|
|
||||||
|
|
||||||
|
|
||||||
class FakeDocument:
|
|
||||||
page_count = 2
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, *_args):
|
|
||||||
return None
|
|
||||||
|
|
||||||
def load_page(self, _index):
|
|
||||||
return FakePage()
|
|
||||||
|
|
||||||
|
|
||||||
def test_scanned_pdf_ocr_preserves_page_order_and_reports_progress(monkeypatch):
|
|
||||||
fake_pymupdf = SimpleNamespace(
|
|
||||||
open=lambda _path: FakeDocument(),
|
|
||||||
Matrix=lambda x, y: (x, y),
|
|
||||||
csRGB="rgb",
|
|
||||||
)
|
|
||||||
monkeypatch.setitem(__import__("sys").modules, "pymupdf", fake_pymupdf)
|
|
||||||
progress = []
|
|
||||||
reservation = SimpleNamespace()
|
|
||||||
config = SimpleNamespace(
|
|
||||||
api_key="configured",
|
|
||||||
ocr_model="qwen-vl-ocr",
|
|
||||||
vision_model="vision",
|
|
||||||
vision_max_tokens=2048,
|
|
||||||
)
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("services.pdf_ocr_service.get_chat_model_config", return_value=config),
|
|
||||||
patch("services.pdf_ocr_service.prepare_image", return_value=SimpleNamespace()),
|
|
||||||
patch(
|
|
||||||
"services.pdf_ocr_service.call_vision_model",
|
|
||||||
side_effect=[
|
|
||||||
{"content": "第一页文字", "usage": {"total_tokens": 10}},
|
|
||||||
{"content": "第二页文字", "usage": {"total_tokens": 12}},
|
|
||||||
],
|
|
||||||
),
|
|
||||||
patch("services.pdf_ocr_service.reserve_avatar_tokens", return_value=reservation) as reserve,
|
|
||||||
patch("services.pdf_ocr_service.settle_reservation") as settle,
|
|
||||||
):
|
|
||||||
text = extract_scanned_pdf_text(
|
|
||||||
MagicMock(),
|
|
||||||
SimpleNamespace(id="avatar-1"),
|
|
||||||
"/tmp/scanned.pdf",
|
|
||||||
on_progress=lambda done, total: progress.append((done, total)),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert text == "[第 1 页]\n第一页文字\n\n[第 2 页]\n第二页文字"
|
|
||||||
assert progress == [(1, 2), (2, 2)]
|
|
||||||
assert reserve.call_count == 2
|
|
||||||
assert settle.call_count == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_scanned_pdf_ocr_releases_tokens_after_retries_fail(monkeypatch):
|
|
||||||
fake_document = FakeDocument()
|
|
||||||
fake_document.page_count = 1
|
|
||||||
fake_pymupdf = SimpleNamespace(
|
|
||||||
open=lambda _path: fake_document,
|
|
||||||
Matrix=lambda x, y: (x, y),
|
|
||||||
csRGB="rgb",
|
|
||||||
)
|
|
||||||
monkeypatch.setitem(__import__("sys").modules, "pymupdf", fake_pymupdf)
|
|
||||||
monkeypatch.setenv("KNOWLEDGE_PDF_OCR_ATTEMPTS", "2")
|
|
||||||
reservation = SimpleNamespace()
|
|
||||||
config = SimpleNamespace(
|
|
||||||
api_key="configured",
|
|
||||||
ocr_model="qwen-vl-ocr",
|
|
||||||
vision_model="vision",
|
|
||||||
vision_max_tokens=2048,
|
|
||||||
)
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("services.pdf_ocr_service.get_chat_model_config", return_value=config),
|
|
||||||
patch("services.pdf_ocr_service.prepare_image", return_value=SimpleNamespace()),
|
|
||||||
patch("services.pdf_ocr_service.call_vision_model", side_effect=RuntimeError("timeout")) as call,
|
|
||||||
patch("services.pdf_ocr_service.reserve_avatar_tokens", return_value=reservation),
|
|
||||||
patch("services.pdf_ocr_service.release_reservation") as release,
|
|
||||||
patch("services.pdf_ocr_service.time.sleep"),
|
|
||||||
):
|
|
||||||
with pytest.raises(RuntimeError, match="第 1/1 页识别失败"):
|
|
||||||
extract_scanned_pdf_text(MagicMock(), SimpleNamespace(id="avatar-1"), "/tmp/scanned.pdf")
|
|
||||||
|
|
||||||
assert call.call_count == 2
|
|
||||||
release.assert_called_once()
|
|
||||||
@@ -1,56 +1,42 @@
|
|||||||
# 会会数字分身 —— Docker 测试实例(独立端口,不干扰现有 :8088 huihui 部署)
|
# 会会数字分身 —— Docker 测试实例(独立端口,不干扰现有 :8088 huihui 部署)
|
||||||
services:
|
services:
|
||||||
avatar-backend:
|
avatar-backend:
|
||||||
build:
|
build: ./backend
|
||||||
context: ./backend
|
image: avatar-test-backend:latest
|
||||||
args:
|
|
||||||
APP_GIT_SHA: ${APP_GIT_SHA:?APP_GIT_SHA must be the full release commit}
|
|
||||||
APP_BUILD_TIME: ${APP_BUILD_TIME:?APP_BUILD_TIME must be set}
|
|
||||||
image: avatar-test-backend:${APP_GIT_SHA}
|
|
||||||
container_name: avatar-test-backend
|
container_name: avatar-test-backend
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
environment:
|
||||||
DATABASE_URL: sqlite:////data/db/avatar.db
|
DATABASE_URL: sqlite:////data/avatar.db
|
||||||
UPLOAD_DIR: /data/uploads
|
UPLOAD_DIR: /data/uploads
|
||||||
CHAT_MODEL_CONFIG_URL: http://host.docker.internal:8000/api/ai-models/runtime/digital-avatar
|
CHAT_MODEL_CONFIG_URL: http://host.docker.internal:8000/api/ai-models/runtime/digital-avatar
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
- "host.docker.internal:host-gateway"
|
- "host.docker.internal:host-gateway"
|
||||||
volumes:
|
volumes:
|
||||||
# Mount the directory, not only avatar.db: SQLite WAL/SHM files must survive recreation.
|
- avatar-data:/data
|
||||||
- ${AVATAR_DB_DIR:?AVATAR_DB_DIR must contain the persistent avatar.db}:/data/db
|
|
||||||
- ${AVATAR_UPLOAD_DIR:?AVATAR_UPLOAD_DIR must point to persistent uploads}:/data/uploads
|
|
||||||
expose:
|
expose:
|
||||||
- "8000"
|
- "8000"
|
||||||
ports:
|
ports:
|
||||||
- "8011:8000" # 仅用于直接调试 API;前端经内部网络访问,不走 host 端口
|
- "8011:8000" # 仅用于直接调试 API;前端经内部网络访问,不走 host 端口
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "python", "-c", "import json,urllib.request; d=json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=5))['data']; assert d['status']=='ok' and all(d['checks'].values())"]
|
|
||||||
interval: 10s
|
|
||||||
timeout: 8s
|
|
||||||
retries: 12
|
|
||||||
start_period: 20s
|
|
||||||
networks:
|
networks:
|
||||||
- avatar-net
|
- avatar-net
|
||||||
|
|
||||||
avatar-frontend:
|
avatar-frontend:
|
||||||
build:
|
build: .
|
||||||
context: .
|
image: avatar-test-frontend:latest
|
||||||
args:
|
|
||||||
APP_GIT_SHA: ${APP_GIT_SHA:?APP_GIT_SHA must be the full release commit}
|
|
||||||
APP_BUILD_TIME: ${APP_BUILD_TIME:?APP_BUILD_TIME must be set}
|
|
||||||
image: avatar-test-frontend:${APP_GIT_SHA}
|
|
||||||
container_name: avatar-test-frontend
|
container_name: avatar-test-frontend
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
- "8099:80" # 浏览器访问 http://<host>:8099
|
- "8099:80" # 浏览器访问 http://<host>:8099
|
||||||
depends_on:
|
depends_on:
|
||||||
avatar-backend:
|
- avatar-backend
|
||||||
condition: service_healthy
|
|
||||||
networks:
|
networks:
|
||||||
- avatar-net
|
- avatar-net
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
avatar-net:
|
avatar-net:
|
||||||
driver: bridge
|
driver: bridge
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
avatar-data:
|
||||||
|
|||||||
@@ -61,9 +61,7 @@ WECHAT_VIRTUAL_PRODUCT_2=<100元套餐商品ID>
|
|||||||
WECHAT_VIRTUAL_PRODUCT_3=<1000元套餐商品ID>
|
WECHAT_VIRTUAL_PRODUCT_3=<1000元套餐商品ID>
|
||||||
WECHAT_VIRTUAL_PRODUCT_4=<10000元套餐商品ID>
|
WECHAT_VIRTUAL_PRODUCT_4=<10000元套餐商品ID>
|
||||||
|
|
||||||
AVATAR_DB_DIR=/srv/digital-avatar/data/db
|
DATABASE_URL=sqlite:////data/avatar.db
|
||||||
AVATAR_UPLOAD_DIR=/srv/digital-avatar/data/uploads
|
|
||||||
DATABASE_URL=sqlite:////data/db/avatar.db
|
|
||||||
UPLOAD_DIR=/data/uploads
|
UPLOAD_DIR=/data/uploads
|
||||||
CHAT_MODEL_CONFIG_URL=http://<huihuisquare-api>/api/ai-models/runtime/digital-avatar
|
CHAT_MODEL_CONFIG_URL=http://<huihuisquare-api>/api/ai-models/runtime/digital-avatar
|
||||||
EMBEDDING_API_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
|
EMBEDDING_API_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||||
@@ -76,17 +74,13 @@ VISION_OCR_MODEL=qwen-vl-ocr
|
|||||||
VISION_MAX_OUTPUT_TOKENS=2048
|
VISION_MAX_OUTPUT_TOKENS=2048
|
||||||
VISION_TIMEOUT_SECONDS=90
|
VISION_TIMEOUT_SECONDS=90
|
||||||
VISION_TOKEN_RESERVE=12000
|
VISION_TOKEN_RESERVE=12000
|
||||||
APP_GIT_SHA=<本次发布的完整提交SHA>
|
|
||||||
APP_BUILD_TIME=<UTC ISO-8601构建时间>
|
|
||||||
CHAT_IMAGE_MAX_BYTES=8388608
|
CHAT_IMAGE_MAX_BYTES=8388608
|
||||||
CHAT_IMAGE_MAX_PIXELS=16000000
|
CHAT_IMAGE_MAX_PIXELS=16000000
|
||||||
CHAT_ATTACHMENT_RETENTION_HOURS=24
|
CHAT_ATTACHMENT_RETENTION_HOURS=24
|
||||||
CHAT_ATTACHMENT_CLEANUP_MINUTES=60
|
CHAT_ATTACHMENT_CLEANUP_MINUTES=60
|
||||||
```
|
```
|
||||||
|
|
||||||
如生产 AI 配置中心不可用,还应提供当前项目支持的 `OPENAI_API_KEY`、`OPENAI_BASE_URL`、`CHAT_MODEL` 等兜底配置。数据库文件与上传目录必须从宿主机显式挂载,不能存放在容器临时层。
|
如生产 AI 配置中心不可用,还应提供当前项目支持的 `OPENAI_API_KEY`、`OPENAI_BASE_URL`、`CHAT_MODEL` 等兜底配置。`/data` 必须挂载持久卷,数据库与知识库文件不可存放在容器临时层。
|
||||||
|
|
||||||
`AVATAR_DB_DIR` 和 `AVATAR_UPLOAD_DIR` 必须是已备份的宿主机绝对路径,编排缺少任一变量都会直接拒绝构建或启动,防止误挂空卷造成用户、分身或知识库“丢失”的假象。SQLite 必须挂载整个数据库目录,不能只挂载 `avatar.db` 单文件,否则 `avatar.db-wal` 和 `avatar.db-shm` 会留在容器临时层,换容器后可能出现数据状态回退。
|
|
||||||
|
|
||||||
`EMBEDDING_API_URL` 同时支持 OpenAI 兼容基础地址(如上面的 `/v1`)和完整的 `/v1/embeddings` 地址,后端会统一请求 `/embeddings`。发布后必须在后端容器内执行一次最小向量探针,确认返回向量数量和维度,而不能只检查 `/api/health`。
|
`EMBEDDING_API_URL` 同时支持 OpenAI 兼容基础地址(如上面的 `/v1`)和完整的 `/v1/embeddings` 地址,后端会统一请求 `/embeddings`。发布后必须在后端容器内执行一次最小向量探针,确认返回向量数量和维度,而不能只检查 `/api/health`。
|
||||||
|
|
||||||
@@ -103,7 +97,7 @@ App 与 H5 积分充值使用会会支付体系的 `payment-v3/payment/pay`,
|
|||||||
```bash
|
```bash
|
||||||
BACKUP_DIR="backups/$(date +%Y%m%d-%H%M%S)"
|
BACKUP_DIR="backups/$(date +%Y%m%d-%H%M%S)"
|
||||||
mkdir -p "$BACKUP_DIR"
|
mkdir -p "$BACKUP_DIR"
|
||||||
cp /srv/digital-avatar/data/db/avatar.db "$BACKUP_DIR/"
|
cp /srv/digital-avatar/data/avatar.db "$BACKUP_DIR/"
|
||||||
tar -C /srv/digital-avatar/data -czf "$BACKUP_DIR/uploads.tgz" uploads
|
tar -C /srv/digital-avatar/data -czf "$BACKUP_DIR/uploads.tgz" uploads
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -113,22 +107,13 @@ tar -C /srv/digital-avatar/data -czf "$BACKUP_DIR/uploads.tgz" uploads
|
|||||||
git fetch origin
|
git fetch origin
|
||||||
git checkout <已验收的提交SHA>
|
git checkout <已验收的提交SHA>
|
||||||
cd digital-avatar-app
|
cd digital-avatar-app
|
||||||
export APP_GIT_SHA="$(git rev-parse HEAD)"
|
docker compose build --pull avatar-backend avatar-frontend
|
||||||
export APP_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
docker compose up -d avatar-backend avatar-frontend
|
||||||
docker compose build --pull --no-cache avatar-backend avatar-frontend
|
|
||||||
docker compose up -d --force-recreate --wait avatar-backend avatar-frontend
|
|
||||||
docker compose ps
|
docker compose ps
|
||||||
python3 scripts/verify-deployment.py \
|
curl -fsS http://127.0.0.1:8099/api/health
|
||||||
https://digital.99hui.com "$APP_GIT_SHA" \
|
|
||||||
--backend-container avatar-backend \
|
|
||||||
--frontend-container avatar-frontend \
|
|
||||||
--expected-db-source /srv/digital-avatar/data/db \
|
|
||||||
--expected-upload-source /srv/digital-avatar/data/uploads
|
|
||||||
docker compose exec avatar-backend python -c 'import embeddings; v=embeddings.embed(["部署向量探针"]); print(len(v), len(v[0]))'
|
docker compose exec avatar-backend python -c 'import embeddings; v=embeddings.embed(["部署向量探针"]); print(len(v), len(v[0]))'
|
||||||
```
|
```
|
||||||
|
|
||||||
Jenkins 必须以 `verify-deployment.py` 返回成功作为发布成功条件,不能只以镜像构建或容器启动成功作为条件。脚本会同时核对公网前后端 Git SHA、数据库可读、上传目录可写、PDF OCR 依赖和宿主机数据挂载;任意一项不一致都会返回非零状态并阻止发布标绿。镜像使用 Git SHA 标签,不再依赖可被旧缓存覆盖的 `latest`。
|
|
||||||
|
|
||||||
生产编排应把示例中的测试端口改为内网暴露,由统一 HTTPS 网关接入。后端暂时使用 SQLite,必须保持单实例写入;若扩展为多后端实例,应先迁移到 PostgreSQL,并把延迟接管任务改为共享队列。
|
生产编排应把示例中的测试端口改为内网暴露,由统一 HTTPS 网关接入。后端暂时使用 SQLite,必须保持单实例写入;若扩展为多后端实例,应先迁移到 PostgreSQL,并把延迟接管任务改为共享队列。
|
||||||
|
|
||||||
## 4. 网关要求
|
## 4. 网关要求
|
||||||
@@ -168,7 +153,7 @@ location /api/ {
|
|||||||
5. 使用过期或伪造 token 时进入登录页并显示凭证失效,不得继续访问旧用户数据。
|
5. 使用过期或伪造 token 时进入登录页并显示凭证失效,不得继续访问旧用户数据。
|
||||||
6. 分身聊天 SSE 逐段输出正常,Markdown 正常渲染,知识库优先级和积分扣费正常。
|
6. 分身聊天 SSE 逐段输出正常,Markdown 正常渲染,知识库优先级和积分扣费正常。
|
||||||
7. 开启 BOXIM 主动接管后保持在线,默认三分钟回复、自定义等待时间、已读回执、分身防回环和主人发言暂停均正常。
|
7. 开启 BOXIM 主动接管后保持在线,默认三分钟回复、自定义等待时间、已读回执、分身防回环和主人发言暂停均正常。
|
||||||
8. 重建容器后数据库、头像、知识库文档仍存在,`/api/health` 的 `gitSha` 与发布 SHA 一致,`database`、`uploads`、`pdfOcr` 三项检查均为 `true`。
|
8. 重建容器后数据库、头像、知识库文档仍存在,`/api/health` 返回成功。
|
||||||
9. `https://digital.99hui.com/api/health` 可访问,证书域名和有效期正确,HTTP 自动跳转 HTTPS。
|
9. `https://digital.99hui.com/api/health` 可访问,证书域名和有效期正确,HTTP 自动跳转 HTTPS。
|
||||||
10. 微信和支付宝各创建一笔最小套餐订单,未付款时积分不变;支付成功后回调到账一次,重复回调积分不重复增加。
|
10. 微信和支付宝各创建一笔最小套餐订单,未付款时积分不变;支付成功后回调到账一次,重复回调积分不重复增加。
|
||||||
11. 微信虚拟支付在沙箱环境完成下单、支付回调、查单兜底和退款回调;错误 OpenID、商品、环境或金额均被拒绝。
|
11. 微信虚拟支付在沙箱环境完成下单、支付回调、查单兜底和退款回调;错误 OpenID、商品、环境或金额均被拒绝。
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Fail a deployment unless frontend and backend run the expected release."""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_json(url):
|
|
||||||
with urllib.request.urlopen(url, timeout=20) as response:
|
|
||||||
if response.status != 200:
|
|
||||||
raise RuntimeError(f"{url} returned HTTP {response.status}")
|
|
||||||
return json.load(response)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("base_url", help="Public site URL, for example https://digital.99hui.com")
|
|
||||||
parser.add_argument("expected_sha", help="Full Git commit SHA being deployed")
|
|
||||||
parser.add_argument("--backend-container", help="Backend container name for image and mount checks")
|
|
||||||
parser.add_argument("--frontend-container", help="Frontend container name for image checks")
|
|
||||||
parser.add_argument("--expected-db-source", help="Required host directory mounted for SQLite and its WAL files")
|
|
||||||
parser.add_argument("--expected-upload-source", help="Required host source mounted as the upload directory")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
base_url = args.base_url.rstrip("/")
|
|
||||||
errors = []
|
|
||||||
try:
|
|
||||||
health = fetch_json(f"{base_url}/api/health").get("data") or {}
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"cannot read backend release metadata: {exc}")
|
|
||||||
health = {}
|
|
||||||
try:
|
|
||||||
frontend = fetch_json(f"{base_url}/version.json")
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"cannot read frontend release metadata: {exc}")
|
|
||||||
frontend = {}
|
|
||||||
|
|
||||||
if health.get("status") != "ok":
|
|
||||||
errors.append(f"backend status is {health.get('status')!r}")
|
|
||||||
failed_checks = [name for name, passed in (health.get("checks") or {}).items() if not passed]
|
|
||||||
if failed_checks:
|
|
||||||
errors.append("backend checks failed: " + ", ".join(failed_checks))
|
|
||||||
if health.get("gitSha") != args.expected_sha:
|
|
||||||
errors.append(f"backend SHA is {health.get('gitSha')!r}")
|
|
||||||
if frontend.get("gitSha") != args.expected_sha:
|
|
||||||
errors.append(f"frontend SHA is {frontend.get('gitSha')!r}")
|
|
||||||
|
|
||||||
if args.backend_container:
|
|
||||||
backend = inspect_container(args.backend_container, errors)
|
|
||||||
check_container_revision(backend, args.expected_sha, "backend", errors)
|
|
||||||
check_mount(backend, args.expected_db_source, "database", errors)
|
|
||||||
check_mount(backend, args.expected_upload_source, "uploads", errors)
|
|
||||||
elif args.expected_db_source or args.expected_upload_source:
|
|
||||||
errors.append("--backend-container is required when checking data mounts")
|
|
||||||
|
|
||||||
if args.frontend_container:
|
|
||||||
frontend_container = inspect_container(args.frontend_container, errors)
|
|
||||||
check_container_revision(frontend_container, args.expected_sha, "frontend", errors)
|
|
||||||
|
|
||||||
if errors:
|
|
||||||
print("Deployment verification failed:", file=sys.stderr)
|
|
||||||
for error in errors:
|
|
||||||
print(f"- {error}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f"Deployment verified: {args.expected_sha}")
|
|
||||||
print("Backend checks: database, uploads, pdfOcr")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def inspect_container(name, errors):
|
|
||||||
try:
|
|
||||||
output = subprocess.check_output(
|
|
||||||
["docker", "inspect", name], universal_newlines=True
|
|
||||||
)
|
|
||||||
return json.loads(output)[0]
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"cannot inspect container {name!r}: {exc}")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def check_container_revision(container, expected_sha, label, errors):
|
|
||||||
actual = ((container.get("Config") or {}).get("Labels") or {}).get(
|
|
||||||
"org.opencontainers.image.revision"
|
|
||||||
)
|
|
||||||
if actual != expected_sha:
|
|
||||||
errors.append(f"{label} container image SHA is {actual!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def check_mount(container, expected_source, label, errors):
|
|
||||||
if not expected_source:
|
|
||||||
return
|
|
||||||
expected = os.path.realpath(expected_source)
|
|
||||||
sources = {
|
|
||||||
os.path.realpath(mount.get("Source", ""))
|
|
||||||
for mount in container.get("Mounts") or []
|
|
||||||
}
|
|
||||||
if expected not in sources:
|
|
||||||
errors.append(f"{label} mount source {expected!r} is not attached")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -147,7 +147,7 @@ const documentState = (doc: any) => {
|
|||||||
if (['uploaded', 'parsing'].includes(String(doc.status || '').toLowerCase())) {
|
if (['uploaded', 'parsing'].includes(String(doc.status || '').toLowerCase())) {
|
||||||
const stage = String(doc.indexStage || 'queued').toLowerCase()
|
const stage = String(doc.indexStage || 'queued').toLowerCase()
|
||||||
const labels: Record<string, string> = {
|
const labels: Record<string, string> = {
|
||||||
queued: '等待处理', extracting: '解析文档', ocr: '扫描件识别', chunking: '切分文本', embedding: '向量化中'
|
queued: '等待处理', extracting: '解析文档', chunking: '切分文本', embedding: '向量化中'
|
||||||
}
|
}
|
||||||
const progress = Math.max(0, Math.min(99, Number(doc.indexProgress || 0)))
|
const progress = Math.max(0, Math.min(99, Number(doc.indexProgress || 0)))
|
||||||
return { tone: 'pending', label: labels[stage] || '处理中', detail: `${labels[stage] || '正在建立知识索引'} ${progress}%`, progress }
|
return { tone: 'pending', label: labels[stage] || '处理中', detail: `${labels[stage] || '正在建立知识索引'} ${progress}%`, progress }
|
||||||
|
|||||||
Reference in New Issue
Block a user