fix(avatar): index knowledge documents asynchronously

digital-avatar-app/backend/routers/knowledge.py

@@ -1,8 +1,5 @@
 import os
-import json
-import logging
 import uuid
-from datetime import datetime, timezone
 
 from fastapi import APIRouter, UploadFile, File, Depends, Header, HTTPException
 from pydantic import BaseModel
@@ -12,10 +9,9 @@ from database import get_db
 from models import KnowledgeDoc, QAPair, KnowledgeChunk, Avatar, User
 from responses import ok, fail
 import embeddings
+from services.knowledge_vectorizer import knowledge_vectorizer
 
 router = APIRouter()
-logger = logging.getLogger(__name__)
-
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 UPLOAD_DIR = os.path.abspath(os.getenv("UPLOAD_DIR", os.path.join(BASE_DIR, "uploads")))
 os.makedirs(UPLOAD_DIR, exist_ok=True)
@@ -71,15 +67,6 @@ def list_docs(avatar_id: str, authorization: str = Header(None), db: Session = D
         .order_by(KnowledgeDoc.created_at.desc())
         .all()
     )
-    # Older synchronous uploads could be interrupted after persisting "parsing".
-    # New uploads are committed only after indexing finishes, so these rows are stale.
-    stale_docs = [doc for doc in docs if doc.status == "parsing"]
-    if stale_docs:
-        for doc in stale_docs:
-            doc.status = "failed"
-            doc.vectorized = False
-            doc.chunk_count = 0
-        db.commit()
     return ok([_doc_payload(d) for d in docs])
 
 
@@ -108,50 +95,42 @@ async def upload_doc(avatar_id: str, file: UploadFile = File(...), authorization
         status="parsing",
     )
 
-    # Complete extraction and embedding before the first database commit so a
-    # process restart cannot leave a permanent "parsing" row behind.
-    try:
-        text = embeddings.extract_text(path, ext)
-        chunks = embeddings.chunk_text(text)
-        if not chunks:
-            raise ValueError("文档没有可建立索引的文字内容")
-        vectors = embeddings.embed(chunks)
-        if len(vectors) != len(chunks):
-            raise ValueError("向量服务返回数量与文档分段不一致")
-        doc.vectorized = True
-        doc.embedding_model = embeddings.MODEL
-        doc.chunk_count = len(chunks)
-        doc.vectorized_at = datetime.now(timezone.utc)
-        doc.status = "ready"
-        db.add(doc)
-        for i, (chunk, vector) in enumerate(zip(chunks, vectors)):
-            db.add(
-                KnowledgeChunk(
-                    doc_id=doc.id,
-                    avatar_id=avatar_id,
-                    content=chunk,
-                    vector=json.dumps(vector),
-                    chunk_index=i,
-                    embedding_model=embeddings.MODEL,
-                )
-            )
-        db.commit()
-        db.refresh(doc)
-    except Exception as exc:
-        db.rollback()
-        doc.status = "failed"
-        doc.vectorized = False
-        doc.embedding_model = ""
-        doc.chunk_count = 0
-        doc.vectorized_at = None
-        db.add(doc)
-        db.commit()
-        db.refresh(doc)
-        logger.exception("knowledge vectorization failed for %s: %s", doc.id, exc)
+    # Persist and acknowledge the upload first. Extraction and embeddings may take
+    # minutes for a PDF and must never consume the browser request timeout.
+    db.add(doc)
+    db.commit()
+    db.refresh(doc)
+    knowledge_vectorizer.enqueue(doc.id)
 
     return ok(_doc_payload(doc))
 
 
+@router.post("/avatar/{avatar_id}/knowledge/docs/{doc_id}/retry")
+def retry_doc(avatar_id: str, doc_id: str, authorization: str = Header(None), db: Session = Depends(get_db)):
+    _require_owned_avatar(db, avatar_id, authorization)
+    doc = db.query(KnowledgeDoc).filter(
+        KnowledgeDoc.id == doc_id, KnowledgeDoc.avatar_id == avatar_id
+    ).first()
+    if not doc:
+        return fail("文档不存在", code=404)
+    if doc.vectorized and doc.status == "ready":
+        return ok(_doc_payload(doc))
+    stored_name = os.path.basename(doc.file_url or "")
+    if not stored_name or not os.path.isfile(os.path.join(UPLOAD_DIR, avatar_id, stored_name)):
+        return fail("原文件不可用，请重新上传", code=400)
+    db.query(KnowledgeChunk).filter(KnowledgeChunk.doc_id == doc.id).delete()
+    doc.status = "parsing"
+    doc.vectorized = False
+    doc.embedding_model = ""
+    doc.chunk_count = 0
+    doc.vectorized_at = None
+    doc.error_message = ""
+    db.commit()
+    db.refresh(doc)
+    knowledge_vectorizer.enqueue(doc.id)
+    return ok(_doc_payload(doc))
+
+
 @router.delete("/avatar/{avatar_id}/knowledge/docs/{doc_id}")
 def delete_doc(avatar_id: str, doc_id: str, authorization: str = Header(None), db: Session = Depends(get_db)):
     _require_owned_avatar(db, avatar_id, authorization)