Compare commits

..
8 changed files with 802 additions and 49 deletions
+188 -1
View File
@@ -1,16 +1,24 @@
"""数字分身管理服务层 — 同步连接数字分身应用的 SQLite 数据库"""
import json
import os
from datetime import datetime, timedelta
from typing import Optional, Tuple
from sqlalchemy import create_engine, text
from sqlalchemy import create_engine, select, text
from sqlalchemy.orm import sessionmaker, Session
from app.core.config import settings
from app.core.logger import logger
from app.models import UserPersonality, VirtualUser
_engine = None
_SessionLocal: Optional[sessionmaker] = None
AVATAR_ACCOUNT_PREFIX = "__avatar__:"
SQUARE_INTERACTION_PERMISSION = "interact"
SQUARE_INTERACTION_ACTIONS = frozenset({"like", "collect", "comment", "reply"})
def _get_engine_and_session():
global _engine, _SessionLocal
@@ -66,6 +74,185 @@ def _get_global_token_balance(db: Session) -> int:
return 0
def _decode_config(value) -> dict:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
decoded = json.loads(value)
return decoded if isinstance(decoded, dict) else {}
except (json.JSONDecodeError, ValueError):
return {}
return {}
def is_delegated_avatar_user(user: VirtualUser | None) -> bool:
return bool(user and (user.account or "").startswith(AVATAR_ACCOUNT_PREFIX))
def delegated_avatar_id(user: VirtualUser | None) -> str:
if not is_delegated_avatar_user(user):
return ""
return (user.account or "")[len(AVATAR_ACCOUNT_PREFIX):]
def _list_square_interaction_authorizations(db: Session) -> list[dict]:
"""读取已明确授权分身参与广场互动的身份与会会令牌。"""
rows = db.execute(text("""
SELECT
a.id AS avatar_id,
a.name AS avatar_name,
a.display_name AS avatar_display_name,
a.description AS avatar_description,
a.photo_url AS avatar_photo_url,
a.config AS avatar_config,
u.huihui_user_id,
u.nickname AS owner_nickname,
u.avatar_url AS owner_avatar_url,
u.huihui_token
FROM avatars a
JOIN users u ON u.huihui_user_id = a.owner_id
WHERE a.status = 'active'
""")).fetchall()
authorized = []
for row in rows:
config = _decode_config(row.avatar_config)
permissions = config.get("authorizationPermissions", [])
if not isinstance(permissions, list) or SQUARE_INTERACTION_PERMISSION not in permissions:
continue
platform_uid = str(row.huihui_user_id or "").strip()
token = str(row.huihui_token or "").strip()
if not platform_uid or not token:
continue
authorized.append({
"avatar_id": str(row.avatar_id),
"avatar_name": row.avatar_display_name or row.avatar_name or row.owner_nickname or "数字分身",
"avatar_description": row.avatar_description or "",
"avatar_url": _resolve_photo_url(row.avatar_photo_url or row.owner_avatar_url or ""),
"config": config,
"platform_uid": platform_uid,
"token": token,
})
return authorized
def get_square_interaction_permissions(avatar_id: str) -> frozenset[str]:
"""实时复核授权;数据库不可用、令牌失效或撤权时一律拒绝执行。"""
avatar_db = get_session()
if avatar_db is None:
return frozenset()
try:
authorized_ids = {
item["avatar_id"] for item in _list_square_interaction_authorizations(avatar_db)
}
return SQUARE_INTERACTION_ACTIONS if avatar_id in authorized_ids else frozenset()
except Exception as exc:
logger.error(f"读取数字分身广场互动授权失败: {exc}")
return frozenset()
finally:
avatar_db.close()
def _word_count_range(config: dict) -> tuple[int, int]:
ranges = {
"short": (10, 35),
"medium": (20, 60),
"long": (30, 80),
}
return ranges.get(str(config.get("responseLength") or "medium"), (20, 60))
async def sync_square_interaction_users(db) -> set[str]:
"""把已授权分身同步为调度器身份,并刷新其会会会话。"""
avatar_db = get_session()
if avatar_db is None:
logger.warning("数字分身数据库不可用,跳过广场互动授权同步")
return set()
try:
authorized = _list_square_interaction_authorizations(avatar_db)
except Exception as exc:
logger.error(f"同步数字分身广场互动授权失败: {exc}")
return set()
finally:
avatar_db.close()
from app.core.redis_client import delete_session, set_session
result = await db.execute(
select(VirtualUser).where(VirtualUser.account.like(f"{AVATAR_ACCOUNT_PREFIX}%"))
)
existing_users = {delegated_avatar_id(user): user for user in result.scalars().all()}
authorized_ids = {item["avatar_id"] for item in authorized}
for avatar_id, user in existing_users.items():
if avatar_id not in authorized_ids:
user.is_enabled = 0
user.status = 0
user.session_token = None
user.session_expires_at = None
await delete_session(user.id)
for item in authorized:
avatar_id = item["avatar_id"]
user = existing_users.get(avatar_id)
if user is None:
user = VirtualUser(
nickname=item["avatar_name"],
account=f"{AVATAR_ACCOUNT_PREFIX}{avatar_id}",
password_enc="",
status=2,
is_enabled=1,
platform_uid=item["platform_uid"],
remark="用户授权的数字分身广场互动身份",
)
db.add(user)
await db.flush()
expires_at = datetime.now() + timedelta(days=1)
user.nickname = item["avatar_name"]
user.real_name = item["avatar_name"]
user.avatar_url = item["avatar_url"]
user.platform_uid = item["platform_uid"]
user.session_token = item["token"]
user.session_expires_at = expires_at
user.last_login_at = datetime.now()
user.status = 2
user.is_enabled = 1
config = item["config"]
personality_result = await db.execute(
select(UserPersonality).where(UserPersonality.user_id == user.id)
)
personality = personality_result.scalar_one_or_none()
word_min, word_max = _word_count_range(config)
prompt_parts = [item["avatar_description"], str(config.get("systemPrompt") or "")]
style_prompt = "\n".join(part.strip() for part in prompt_parts if part and part.strip())
if personality is None:
personality = UserPersonality(user_id=user.id)
db.add(personality)
personality.language_style = str(config.get("replyStyle") or "professional")
personality.personality_desc = item["avatar_description"]
personality.comment_style_prompt = style_prompt
personality.word_count_min = word_min
personality.word_count_max = word_max
await set_session(user.id, {
"token": item["token"],
"session_id": f"avatar:{avatar_id}",
"platform_uid": item["platform_uid"],
"org_id": "",
"login_time": datetime.now().isoformat(),
"nickname": item["avatar_name"],
"real_name": item["avatar_name"],
"avatar": item["avatar_url"],
"delegated_avatar_id": avatar_id,
}, expire=86400)
await db.commit()
return authorized_ids
class AvatarService:
@staticmethod
+117 -16
View File
@@ -23,6 +23,7 @@ class SchedulerService:
from app.core.database import AsyncSessionLocal
logger.info("⚡ 立即触发互动任务")
async with AsyncSessionLocal() as session:
await self._sync_delegated_avatar_users(session)
try:
max_concurrent = int(await self._get_config(session, "max_concurrent_users", "5"))
except (TypeError, ValueError):
@@ -146,7 +147,9 @@ class SchedulerService:
async def _check_sessions(self):
"""定时校验登录状态"""
from app.services.news_service import news_service
from app.services.avatar_service import is_delegated_avatar_user
async with AsyncSessionLocal() as db:
await self._sync_delegated_avatar_users(db)
result = await db.execute(
select(VirtualUser).where(VirtualUser.status == 2, VirtualUser.is_enabled == 1)
)
@@ -154,7 +157,7 @@ class SchedulerService:
for user in users:
try:
valid = await news_service.check_session(db, user)
if not valid:
if not valid and not is_delegated_avatar_user(user):
logger.warning(f"用户 {user.account} 会话失效,尝试重登")
await news_service.login(db, user)
except Exception as e:
@@ -163,6 +166,7 @@ class SchedulerService:
async def _run_interactions(self):
"""执行互动任务"""
async with AsyncSessionLocal() as db:
await self._sync_delegated_avatar_users(db)
# 检查调度器开关
enabled = await self._get_config(db, "scheduler_enabled", "true")
if enabled != "true":
@@ -184,8 +188,11 @@ class SchedulerService:
logger.debug(f"[调度] 当前北京时间 {now_time} 不在互动时段 {start_str}-{end_str}")
return
# 获取最小互动间隔(秒)
min_interval = int(await self._get_config(db, "interact_min_interval", "300"))
# 获取互动间隔范围(秒),与调度设置页面字段保持一致
min_interval = await self._get_int_config(db, "interact_interval_min", 300)
max_interval = await self._get_int_config(db, "interact_interval_max", min_interval)
min_interval = max(0, min_interval)
max_interval = max(min_interval, max_interval)
# 获取最大并发
max_concurrent = int(await self._get_config(db, "max_concurrent_users", "5"))
@@ -204,7 +211,7 @@ class SchedulerService:
await self._try_login_users(db)
return
# 检查互动间隔:过滤掉最近 min_interval 秒内已互动的用户
# 每个用户在其最小/最大间隔内取得稳定随机值,直到下次互动后再变化
now_dt = datetime.now()
eligible = []
for u in all_users:
@@ -212,11 +219,17 @@ class SchedulerService:
eligible.append(u)
else:
elapsed = (now_dt - u.last_interact_at).total_seconds()
if elapsed >= min_interval:
interval = random.Random(
f"{u.id}:{u.last_interact_at.isoformat()}"
).randint(min_interval, max_interval)
if elapsed >= interval:
eligible.append(u)
if not eligible:
logger.debug(f"[调度] 所有 {len(all_users)} 个用户在 {min_interval}s 内已互动,跳过本次")
logger.debug(
f"[调度] 所有 {len(all_users)} 个用户尚未达到 "
f"{min_interval}-{max_interval}s 随机互动间隔,跳过本次"
)
return
# 按最后互动时间升序排序:最久没互动的用户优先
@@ -257,10 +270,12 @@ class SchedulerService:
async def _try_login_users(self, db):
"""尝试登录未登录的用户"""
from app.services.news_service import news_service
from app.services.avatar_service import AVATAR_ACCOUNT_PREFIX
result = await db.execute(
select(VirtualUser).where(
VirtualUser.status.in_([0, 3]),
VirtualUser.is_enabled == 1
VirtualUser.is_enabled == 1,
~VirtualUser.account.like(f"{AVATAR_ACCOUNT_PREFIX}%"),
).limit(3)
)
users = result.scalars().all()
@@ -275,6 +290,11 @@ class SchedulerService:
"""执行单用户互动 - 基于真实接口"""
from app.services.news_service import news_service
from app.services.ai_service import ai_service
from app.services.avatar_service import (
delegated_avatar_id,
get_square_interaction_permissions,
is_delegated_avatar_user,
)
async with AsyncSessionLocal() as db:
try:
@@ -289,6 +309,23 @@ class SchedulerService:
"interactions": [],
}
allowed_actions = {"like", "collect", "comment", "reply", "forward"}
if is_delegated_avatar_user(user):
allowed_actions = set(
get_square_interaction_permissions(delegated_avatar_id(user))
)
if not allowed_actions:
user.status = 0
user.is_enabled = 0
await db.commit()
return {
"user_id": user.id,
"account": user.account,
"status": "skipped",
"reason": "avatar_interaction_not_authorized",
"interactions": [],
}
# 检查今日评论限额
can_comment = True
if user.today_comment_count >= user.daily_comment_limit:
@@ -398,14 +435,53 @@ class SchedulerService:
interactions_done = []
action_failures = []
# ① 先记录阅读(每次必做,模拟真实用户打开文章)
done_on_this = today_done.get(news_id, set())
wants = {
"like": (
"like" in allowed_actions
and "like" not in done_on_this
and random.random() < like_prob
),
"collect": (
"collect" in allowed_actions
and "collect" not in done_on_this
and random.random() < collect_prob
),
"forward": (
"forward" in allowed_actions
and "forward" not in done_on_this
and random.random() < forward_prob
),
"reply": (
"reply" in allowed_actions
and can_comment
and personality is not None
and random.random() < reply_prob
),
"comment": (
"comment" in allowed_actions
and can_comment
and personality is not None
and not already_commented_this
and random.random() < comment_prob
),
}
if not any(wants.values()):
return {
"user_id": user.id,
"account": user.account,
"status": "skipped",
"reason": "no_actions_triggered",
"interactions": [],
"article_id": news_id,
"article_title": news_title,
}
# 只有动作命中调度概率后才打开文章
await news_service.read_news(db, user, news_id)
# 今日已对此文章做过的互动类型
done_on_this = today_done.get(news_id, set())
# ② 点赞(每篇文章每用户每天只点赞一次)
if "like" not in done_on_this and random.random() < like_prob:
if wants["like"]:
success, err = await news_service.like_news(db, user, news_id, org_id=article_org_id, to_user_id=news_author, title=news_title)
await self._save_record(db, user, news_id, news_title, "like", None, 0, success, err)
if success:
@@ -415,16 +491,17 @@ class SchedulerService:
action_failures.append({"type": "like", "error": err})
# ③ 收藏(每篇文章每用户每天只收藏一次)
if "collect" not in done_on_this and random.random() < collect_prob:
if wants["collect"]:
success, err = await news_service.collect_news(db, user, news_id, org_id=article_org_id, to_user_id=news_author, title=news_title)
await self._save_record(db, user, news_id, news_title, "collect", None, 0, success, err)
if success:
interactions_done.append("collect")
await self._incr_total(db, user_id)
else:
action_failures.append({"type": "collect", "error": err})
# ④ 转发(每篇文章每用户每天只转发一次)
if "forward" not in done_on_this and random.random() < forward_prob:
if wants["forward"]:
success, err = await news_service.forward_news(db, user, news_id)
await self._save_record(db, user, news_id, news_title, "forward", None, 0, success, err)
if success:
@@ -438,7 +515,7 @@ class SchedulerService:
style_prompt = personality.comment_style_prompt or ""
safe_word_max = min(personality.word_count_max, 80)
if random.random() < reply_prob:
if wants["reply"]:
reply_actions, reply_failures = await self._run_reply_interaction_chain(
db=db,
starter=user,
@@ -455,7 +532,7 @@ class SchedulerService:
action_failures.extend(reply_failures)
# 每篇文章每个用户每天只发一条顶层评论;回复不再要求先评论
if not already_commented_this and random.random() < comment_prob:
if wants["comment"]:
comment_text, tokens = await ai_service.generate_comment(
db, news_title, news_content,
style_prompt, personality.word_count_min, safe_word_max
@@ -679,6 +756,7 @@ class SchedulerService:
async with AsyncSessionLocal() as db:
try:
await self._sync_delegated_avatar_users(db)
now = datetime.now()
await db.execute(
update(PendingReplyTask)
@@ -706,6 +784,12 @@ class SchedulerService:
logger.error(f"待发送回复队列处理异常: {e}")
async def _process_pending_reply_task(self, db, task: PendingReplyTask, news_service, ai_service):
from app.services.avatar_service import (
delegated_avatar_id,
get_square_interaction_permissions,
is_delegated_avatar_user,
)
task.status = 1
task.locked_at = datetime.now()
task.attempts = (task.attempts or 0) + 1
@@ -716,6 +800,13 @@ class SchedulerService:
task.status = 3
task.last_error = "用户未登录或已禁用"
return
if (
is_delegated_avatar_user(actor)
and "reply" not in get_square_interaction_permissions(delegated_avatar_id(actor))
):
task.status = 3
task.last_error = "数字分身广场互动授权已撤销"
return
reply_result = await self._post_contextual_reply(
db=db,
@@ -858,6 +949,16 @@ class SchedulerService:
except (TypeError, ValueError):
return default
async def _sync_delegated_avatar_users(self, db):
from app.services.avatar_service import sync_square_interaction_users
try:
return await sync_square_interaction_users(db)
except Exception as exc:
await db.rollback()
logger.error(f"数字分身广场互动身份同步异常: {exc}")
return set()
async def _incr_total(self, db, user_id: int):
await db.execute(
update(VirtualUser).where(VirtualUser.id == user_id).values(
@@ -0,0 +1,126 @@
import json
import os
import sqlite3
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from app.services import avatar_service
class AvatarSquareAuthorizationTests(unittest.TestCase):
def setUp(self):
fd, self.db_path = tempfile.mkstemp(suffix=".db")
os.close(fd)
connection = sqlite3.connect(self.db_path)
connection.executescript("""
CREATE TABLE users (
huihui_user_id TEXT,
nickname TEXT,
avatar_url TEXT,
huihui_token TEXT
);
CREATE TABLE avatars (
id TEXT,
owner_id TEXT,
name TEXT,
display_name TEXT,
description TEXT,
photo_url TEXT,
config TEXT,
status TEXT
);
""")
connection.execute(
"INSERT INTO users VALUES (?, ?, ?, ?)",
("huihui-7", "主人", "/owner.jpg", "huihui-token"),
)
connection.commit()
connection.close()
avatar_service._engine = None
avatar_service._SessionLocal = None
self.path_patch = patch.object(avatar_service.settings, "AVATAR_DB_PATH", self.db_path)
self.path_patch.start()
def tearDown(self):
self.path_patch.stop()
if avatar_service._engine is not None:
avatar_service._engine.dispose()
avatar_service._engine = None
avatar_service._SessionLocal = None
os.unlink(self.db_path)
def _insert_avatar(self, permissions, *, status="active", token=None):
connection = sqlite3.connect(self.db_path)
connection.execute(
"INSERT INTO avatars VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(
"avatar-7",
"huihui-7",
"avatar",
"小会",
"语气友好,表达简洁",
"/avatar.jpg",
json.dumps({
"authorizationPermissions": permissions,
"replyStyle": "warm",
"responseLength": "short",
}),
status,
),
)
if token is not None:
connection.execute(
"UPDATE users SET huihui_token = ? WHERE huihui_user_id = ?",
(token, "huihui-7"),
)
connection.commit()
connection.close()
def test_interact_permission_exposes_only_requested_square_actions(self):
self._insert_avatar(["chat", "interact"])
permissions = avatar_service.get_square_interaction_permissions("avatar-7")
self.assertEqual(
permissions,
frozenset({"like", "collect", "comment", "reply"}),
)
self.assertNotIn("forward", permissions)
def test_missing_permission_inactive_avatar_or_missing_token_denies_execution(self):
scenarios = [
(["chat"], "active", "huihui-token"),
(["interact"], "inactive", "huihui-token"),
(["interact"], "active", ""),
]
for permissions, status, token in scenarios:
with self.subTest(permissions=permissions, status=status, token=token):
connection = sqlite3.connect(self.db_path)
connection.execute("DELETE FROM avatars")
connection.commit()
connection.close()
self._insert_avatar(permissions, status=status, token=token)
self.assertEqual(
avatar_service.get_square_interaction_permissions("avatar-7"),
frozenset(),
)
def test_delegated_avatar_identity_is_recognized_without_matching_normal_users(self):
delegated = SimpleNamespace(account="__avatar__:avatar-7")
normal = SimpleNamespace(account="13800000000")
self.assertTrue(avatar_service.is_delegated_avatar_user(delegated))
self.assertEqual(avatar_service.delegated_avatar_id(delegated), "avatar-7")
self.assertFalse(avatar_service.is_delegated_avatar_user(normal))
self.assertEqual(avatar_service.delegated_avatar_id(normal), "")
def test_response_length_maps_to_scheduler_comment_limits(self):
self.assertEqual(avatar_service._word_count_range({"responseLength": "short"}), (10, 35))
self.assertEqual(avatar_service._word_count_range({"responseLength": "long"}), (30, 80))
self.assertEqual(avatar_service._word_count_range({"responseLength": "unknown"}), (20, 60))
if __name__ == "__main__":
unittest.main()
+192 -19
View File
@@ -1,4 +1,7 @@
import os
import json
import shutil
import time
import uuid
from fastapi import APIRouter, UploadFile, File, Depends, Header, HTTPException
@@ -19,6 +22,9 @@ os.makedirs(UPLOAD_DIR, exist_ok=True)
ALLOWED_EXT = {".md", ".txt", ".pdf", ".doc", ".docx", ".xlsx"}
MAX_UPLOAD_BYTES = 50 * 1024 * 1024
UPLOAD_CHUNK_BYTES = 1024 * 1024
MULTIPART_CHUNK_BYTES = 5 * 1024 * 1024
MULTIPART_ROOT = ".multipart"
MULTIPART_TTL_SECONDS = 24 * 60 * 60
class QAIn(BaseModel):
@@ -31,6 +37,74 @@ class EnabledIn(BaseModel):
enabled: bool = True
class MultipartUploadIn(BaseModel):
filename: str
fileSize: int
totalChunks: int
def _validate_document(filename: str, file_size: int):
ext = os.path.splitext(filename or "")[1].lower()
if ext not in ALLOWED_EXT:
return None, f"不支持的文件类型:{ext or '空'},仅支持 md/txt/pdf/doc/docx/xlsx"
if file_size <= 0:
return None, "文件内容不能为空"
if file_size > MAX_UPLOAD_BYTES:
return None, "文件不能超过 50MB"
return ext, ""
def _multipart_dir(avatar_id: str, upload_id: str) -> str:
safe_avatar_id = os.path.basename(avatar_id)
safe_upload_id = os.path.basename(upload_id)
if (
safe_avatar_id != avatar_id
or safe_upload_id != upload_id
or len(upload_id) != 32
or any(character not in "0123456789abcdef" for character in upload_id)
):
raise HTTPException(status_code=400, detail="上传标识无效")
return os.path.join(UPLOAD_DIR, MULTIPART_ROOT, safe_avatar_id, safe_upload_id)
def _purge_stale_multipart_uploads(avatar_id: str):
avatar_upload_root = os.path.join(UPLOAD_DIR, MULTIPART_ROOT, os.path.basename(avatar_id))
if not os.path.isdir(avatar_upload_root):
return
cutoff = time.time() - MULTIPART_TTL_SECONDS
for entry in os.scandir(avatar_upload_root):
if entry.is_dir(follow_symlinks=False) and entry.stat(follow_symlinks=False).st_mtime < cutoff:
shutil.rmtree(entry.path, ignore_errors=True)
def _read_multipart_metadata(avatar_id: str, upload_id: str) -> tuple[str, dict]:
upload_dir = _multipart_dir(avatar_id, upload_id)
metadata_path = os.path.join(upload_dir, "metadata.json")
if not os.path.isfile(metadata_path):
raise HTTPException(status_code=404, detail="上传任务不存在或已过期")
with open(metadata_path, "r", encoding="utf-8") as stream:
return upload_dir, json.load(stream)
def _create_knowledge_doc(db: Session, avatar_id: str, filename: str, ext: str, file_size: int, stored: str):
doc = KnowledgeDoc(
id=uuid.uuid4().hex,
avatar_id=avatar_id,
filename=filename,
file_type=ext.lstrip("."),
file_size=file_size,
file_url=f"/api/files/{avatar_id}/{stored}",
status="parsing",
index_stage="queued",
index_progress=0,
)
db.add(doc)
db.commit()
db.refresh(doc)
knowledge_vectorizer.enqueue(doc.id)
return doc
def _doc_payload(doc: KnowledgeDoc) -> dict:
payload = doc.to_dict()
stored_name = os.path.basename(doc.file_url or "")
@@ -74,9 +148,9 @@ def list_docs(avatar_id: str, authorization: str = Header(None), db: Session = D
@router.post("/avatar/{avatar_id}/knowledge/docs")
async def upload_doc(avatar_id: str, file: UploadFile = File(...), authorization: str = Header(None), db: Session = Depends(get_db)):
_require_owned_avatar(db, avatar_id, authorization)
ext = os.path.splitext(file.filename or "")[1].lower()
if ext not in ALLOWED_EXT:
return fail(f"不支持的文件类型:{ext or '空'},仅支持 md/txt/pdf/doc/docx/xlsx", code=400)
ext, validation_error = _validate_document(file.filename or "", 1)
if validation_error:
return fail(validation_error, code=400)
avatar_dir = os.path.join(UPLOAD_DIR, avatar_id)
os.makedirs(avatar_dir, exist_ok=True)
stored = f"{uuid.uuid4().hex}{ext}"
@@ -94,25 +168,124 @@ async def upload_doc(avatar_id: str, file: UploadFile = File(...), authorization
if os.path.exists(path):
os.remove(path)
return fail(str(exc), code=400)
doc = KnowledgeDoc(
id=uuid.uuid4().hex,
avatar_id=avatar_id,
filename=file.filename,
file_type=ext.lstrip("."),
file_size=file_size,
file_url=f"/api/files/{avatar_id}/{stored}",
status="parsing",
index_stage="queued",
index_progress=0,
if file_size == 0:
if os.path.exists(path):
os.remove(path)
return fail("文件内容不能为空", code=400)
doc = _create_knowledge_doc(db, avatar_id, file.filename or stored, ext, file_size, stored)
return ok(_doc_payload(doc))
@router.post("/avatar/{avatar_id}/knowledge/uploads")
def create_multipart_upload(
avatar_id: str,
body: MultipartUploadIn,
authorization: str = Header(None),
db: Session = Depends(get_db),
):
_require_owned_avatar(db, avatar_id, authorization)
ext, validation_error = _validate_document(body.filename, body.fileSize)
if validation_error:
return fail(validation_error, code=400)
expected_chunks = (body.fileSize + MULTIPART_CHUNK_BYTES - 1) // MULTIPART_CHUNK_BYTES
if body.totalChunks != expected_chunks:
return fail("文件分片数量不正确", code=400)
_purge_stale_multipart_uploads(avatar_id)
upload_id = uuid.uuid4().hex
upload_dir = _multipart_dir(avatar_id, upload_id)
os.makedirs(upload_dir, exist_ok=False)
metadata = {
"filename": body.filename,
"fileSize": body.fileSize,
"totalChunks": body.totalChunks,
"extension": ext,
}
with open(os.path.join(upload_dir, "metadata.json"), "w", encoding="utf-8") as stream:
json.dump(metadata, stream, ensure_ascii=False)
return ok({"uploadId": upload_id, "chunkSize": MULTIPART_CHUNK_BYTES})
@router.post("/avatar/{avatar_id}/knowledge/uploads/{upload_id}/chunks/{chunk_index}")
async def upload_multipart_chunk(
avatar_id: str,
upload_id: str,
chunk_index: int,
file: UploadFile = File(...),
authorization: str = Header(None),
db: Session = Depends(get_db),
):
_require_owned_avatar(db, avatar_id, authorization)
upload_dir, metadata = _read_multipart_metadata(avatar_id, upload_id)
total_chunks = int(metadata["totalChunks"])
if chunk_index < 0 or chunk_index >= total_chunks:
return fail("文件分片序号不正确", code=400)
expected_size = min(
MULTIPART_CHUNK_BYTES,
int(metadata["fileSize"]) - chunk_index * MULTIPART_CHUNK_BYTES,
)
part_path = os.path.join(upload_dir, f"{chunk_index}.part")
temporary_path = f"{part_path}.uploading"
received = 0
try:
with open(temporary_path, "wb") as stream:
while chunk := await file.read(UPLOAD_CHUNK_BYTES):
received += len(chunk)
if received > expected_size:
raise ValueError("文件分片大小不正确")
stream.write(chunk)
if received != expected_size:
raise ValueError("文件分片大小不正确")
os.replace(temporary_path, part_path)
except ValueError as exc:
if os.path.exists(temporary_path):
os.remove(temporary_path)
return fail(str(exc), code=400)
return ok({"chunkIndex": chunk_index, "uploadedBytes": received})
# Persist and acknowledge the upload first. Extraction and embeddings may take
# minutes for a PDF and must never consume the browser request timeout.
db.add(doc)
db.commit()
db.refresh(doc)
knowledge_vectorizer.enqueue(doc.id)
@router.post("/avatar/{avatar_id}/knowledge/uploads/{upload_id}/complete")
def complete_multipart_upload(
avatar_id: str,
upload_id: str,
authorization: str = Header(None),
db: Session = Depends(get_db),
):
_require_owned_avatar(db, avatar_id, authorization)
upload_dir, metadata = _read_multipart_metadata(avatar_id, upload_id)
total_chunks = int(metadata["totalChunks"])
part_paths = [os.path.join(upload_dir, f"{index}.part") for index in range(total_chunks)]
if not all(os.path.isfile(path) for path in part_paths):
return fail("文件分片尚未上传完整", code=400)
if sum(os.path.getsize(path) for path in part_paths) != int(metadata["fileSize"]):
return fail("文件分片总大小不正确", code=400)
avatar_dir = os.path.join(UPLOAD_DIR, avatar_id)
os.makedirs(avatar_dir, exist_ok=True)
stored = f"{uuid.uuid4().hex}{metadata['extension']}"
final_path = os.path.join(avatar_dir, stored)
temporary_path = f"{final_path}.assembling"
try:
with open(temporary_path, "wb") as output:
for part_path in part_paths:
with open(part_path, "rb") as source:
shutil.copyfileobj(source, output, UPLOAD_CHUNK_BYTES)
os.replace(temporary_path, final_path)
doc = _create_knowledge_doc(
db,
avatar_id,
metadata["filename"],
metadata["extension"],
int(metadata["fileSize"]),
stored,
)
except Exception:
if os.path.exists(temporary_path):
os.remove(temporary_path)
raise
shutil.rmtree(upload_dir, ignore_errors=True)
return ok(_doc_payload(doc))
@@ -87,6 +87,84 @@ def test_upload_rejects_oversize_file_before_queuing_indexing(
assert not list((tmp_path / context["avatar"].id).glob("*"))
def test_multipart_upload_reassembles_file_before_queuing_indexing(
tmp_path: Path,
authorization_context,
):
context = authorization_context
avatar_id = context["avatar"].id
content = b"0123456789"
with (
patch("routers.knowledge.UPLOAD_DIR", str(tmp_path)),
patch("routers.knowledge.MULTIPART_CHUNK_BYTES", 4),
patch("routers.knowledge.knowledge_vectorizer.enqueue") as enqueue,
):
created = client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads",
headers=context["owner_headers"],
json={"filename": "large.pdf", "fileSize": len(content), "totalChunks": 3},
).json()["data"]
for index, chunk in enumerate((content[:4], content[4:8], content[8:])):
response = client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads/{created['uploadId']}/chunks/{index}",
headers=context["owner_headers"],
files={"file": (f"chunk-{index}", chunk, "application/octet-stream")},
)
assert response.json()["code"] == 200
completed = client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads/{created['uploadId']}/complete",
headers=context["owner_headers"],
).json()["data"]
assert completed["status"] == "parsing"
assert completed["fileSize"] == len(content)
enqueue.assert_called_once_with(completed["id"])
stored_path = tmp_path / avatar_id / Path(completed["fileUrl"]).name
assert stored_path.read_bytes() == content
assert not (tmp_path / ".multipart" / avatar_id / created["uploadId"]).exists()
db = SessionLocal()
try:
stored = db.query(KnowledgeDoc).filter(KnowledgeDoc.id == completed["id"]).one()
db.delete(stored)
db.commit()
finally:
db.close()
def test_multipart_upload_rejects_incomplete_parts(
tmp_path: Path,
authorization_context,
):
context = authorization_context
avatar_id = context["avatar"].id
with (
patch("routers.knowledge.UPLOAD_DIR", str(tmp_path)),
patch("routers.knowledge.MULTIPART_CHUNK_BYTES", 4),
patch("routers.knowledge.knowledge_vectorizer.enqueue") as enqueue,
):
created = client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads",
headers=context["owner_headers"],
json={"filename": "large.pdf", "fileSize": 6, "totalChunks": 2},
).json()["data"]
client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads/{created['uploadId']}/chunks/0",
headers=context["owner_headers"],
files={"file": ("chunk-0", b"0123", "application/octet-stream")},
)
response = client.post(
f"/api/avatar/{avatar_id}/knowledge/uploads/{created['uploadId']}/complete",
headers=context["owner_headers"],
)
assert response.json()["code"] == 400
assert response.json()["message"] == "文件分片尚未上传完整"
enqueue.assert_not_called()
def test_background_vectorizer_commits_ready_document_and_chunks_together(
tmp_path: Path,
authorization_context,
+66 -2
View File
@@ -333,12 +333,76 @@ export interface SearchResult {
export const getKnowledgeDocs = (avatarId: string) =>
request.get<KnowledgeDoc[]>(`/avatar/${avatarId}/knowledge/docs`)
// 上传文档(支持 md/txt/pdf/doc/docx/xlsx)
export const uploadKnowledgeDoc = (
const KNOWLEDGE_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024
const uploadKnowledgeChunk = async (
avatarId: string,
uploadId: string,
chunkIndex: number,
chunk: Blob,
onProgress?: (loaded: number) => void
) => {
const form = new FormData()
form.append('file', chunk, `chunk-${chunkIndex}`)
let reportedLoaded = 0
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await request.post(
`/avatar/${avatarId}/knowledge/uploads/${uploadId}/chunks/${chunkIndex}`,
form,
{
headers: { 'Content-Type': 'multipart/form-data' },
timeout: 2 * 60 * 1000,
onUploadProgress: (event) => {
reportedLoaded = Math.max(reportedLoaded, Math.min(event.loaded, chunk.size))
onProgress?.(reportedLoaded)
}
}
)
return
} catch (error: any) {
const status = Number(error?.response?.status || 0)
const retryable = !status || status === 408 || status === 429 || status >= 500
if (!retryable || attempt === 3) throw error
await new Promise((resolve) => window.setTimeout(resolve, attempt * 800))
}
}
}
// 大文件拆成 5MB 分片,避免生产代理的请求体限制拦截整个文件。
export const uploadKnowledgeDoc = async (
avatarId: string,
file: File,
onUploadProgress?: (loaded: number, total: number) => void
) => {
if (file.size > KNOWLEDGE_UPLOAD_CHUNK_SIZE) {
const totalChunks = Math.ceil(file.size / KNOWLEDGE_UPLOAD_CHUNK_SIZE)
const upload: any = await request.post(`/avatar/${avatarId}/knowledge/uploads`, {
filename: file.name,
fileSize: file.size,
totalChunks
})
let uploadedBytes = 0
for (let index = 0; index < totalChunks; index += 1) {
const start = index * KNOWLEDGE_UPLOAD_CHUNK_SIZE
const chunk = file.slice(start, Math.min(start + KNOWLEDGE_UPLOAD_CHUNK_SIZE, file.size))
await uploadKnowledgeChunk(
avatarId,
upload.uploadId,
index,
chunk,
(chunkLoaded) => onUploadProgress?.(uploadedBytes + chunkLoaded, file.size)
)
uploadedBytes += chunk.size
onUploadProgress?.(uploadedBytes, file.size)
}
return request.post<KnowledgeDoc>(
`/avatar/${avatarId}/knowledge/uploads/${upload.uploadId}/complete`,
undefined,
{ timeout: 2 * 60 * 1000 }
)
}
const form = new FormData()
form.append('file', file)
return request.post<KnowledgeDoc>(`/avatar/${avatarId}/knowledge/docs`, form, {
@@ -192,7 +192,7 @@ const permissionItems: Array<{
{
key: 'interact',
title: '广场互动操作',
description: '点赞、收藏、评论、回复等操作',
description: '允许分身代表你点赞、收藏、评论和回复;时段、间隔与触发概率由广场调度设置统一控制',
tone: 'pink',
},
{
@@ -32,7 +32,7 @@
</div>
<div v-if="displayDocs.length" class="mobile-card-list">
<article v-for="doc in displayDocs" :key="doc.id" class="knowledge-card">
<article v-for="doc in displayDocs" :key="doc.id" class="knowledge-card document-card">
<div class="card-icon">{{ fileEmoji(doc.fileType) }}</div>
<div class="card-content">
<div class="card-title-row">
@@ -46,9 +46,14 @@
</div>
</div>
<div class="card-actions">
<button v-if="documentState(doc).tone === 'failed'" class="card-retry" @click="retryDoc(doc.id)">重新索引</button>
<button v-if="!doc.localUploading" class="card-delete" @click="removeDoc(doc.id)">{{ doc.localOnly ? '移除' : '删除' }}</button>
</div>
<div v-if="canRetryDoc(doc)" class="card-retry-area">
<span v-if="retryErrors[doc.id]" class="card-retry-error">{{ retryErrors[doc.id] }}</span>
<button class="card-retry" :disabled="retryingDocs[doc.id]" @click="retryDoc(doc)">
{{ retryingDocs[doc.id] ? '重新索引中…' : '重新索引' }}
</button>
</div>
</article>
</div>
<div v-else class="card-empty">📂 暂无文档,先上传一个知识文件</div>
@@ -115,6 +120,8 @@ const uploading = computed(() => pendingUploads.value.some((doc) => doc.localUpl
const uploadError = ref('')
const dragOver = ref(false)
const fileInput = ref<HTMLInputElement | null>(null)
const retryingDocs = ref<Record<string, boolean>>({})
const retryErrors = ref<Record<string, string>>({})
let documentPollingTimer: ReturnType<typeof setInterval> | undefined
const query = ref('')
@@ -250,14 +257,24 @@ const uploadOne = async (file: File, ext: string) => {
}
}
const retryDoc = async (id: string) => {
if (!avatarId.value) return
uploadError.value = ''
const canRetryDoc = (doc: any) =>
!doc.localOnly && doc.filePresent !== false && documentState(doc).tone === 'failed'
const retryDoc = async (doc: any) => {
if (!avatarId.value || !canRetryDoc(doc) || retryingDocs.value[doc.id]) return
retryingDocs.value = { ...retryingDocs.value, [doc.id]: true }
retryErrors.value = { ...retryErrors.value, [doc.id]: '' }
try {
await retryKnowledgeDoc(avatarId.value, id)
await loadDocs()
const updated: any = await retryKnowledgeDoc(avatarId.value, doc.id)
Object.assign(doc, updated)
startDocumentPolling()
} catch (e: any) {
uploadError.value = e?.message || '重新索引失败'
retryErrors.value = {
...retryErrors.value,
[doc.id]: e?.response?.data?.message || e?.response?.data?.detail || e?.message || '重新索引失败'
}
} finally {
retryingDocs.value = { ...retryingDocs.value, [doc.id]: false }
}
}
@@ -378,6 +395,7 @@ onUnmounted(stopDocumentPolling)
.panel-heading p { margin: -5px 0 0; color: #9398AE; font-size: 12px; }
.mobile-card-list { display: grid; grid-template-columns: minmax(0, 1fr); width: 100%; min-width: 0; gap: 10px; }
.knowledge-card { display: flex; align-items: center; width: 100%; min-width: 0; box-sizing: border-box; gap: 11px; padding: 14px; background: #fff; border: 1px solid #F1E1D3; border-radius: 16px; box-shadow: 0 5px 16px rgba(112, 62, 22, .04); }
.document-card { display: grid; grid-template-columns: 42px minmax(0, 1fr) auto; align-items: center; }
.card-icon { flex: 0 0 auto; width: 42px; height: 42px; display: grid; place-items: center; border-radius: 13px; background: #FFF3E6; font-size: 22px; }
.card-content { min-width: 0; flex: 1; overflow: hidden; }
.card-title-row { display: flex; align-items: center; gap: 8px; min-width: 0; }
@@ -388,10 +406,13 @@ onUnmounted(stopDocumentPolling)
.card-meta, .card-detail { margin: 5px 0 0; color: #9398AE; font-size: 11px; line-height: 1.4; }.card-detail { color: #8B6B58; }
.progress-track { width: 100%; height: 4px; margin-top: 8px; overflow: hidden; border-radius: 999px; background: #FDE7D1; }
.progress-fill { display: block; height: 100%; border-radius: inherit; background: linear-gradient(90deg, #FB923C, #F97316); transition: width .25s ease; }
.card-actions { flex: 0 0 auto; display: flex; flex-direction: column; align-items: stretch; gap: 6px; }
.card-actions { flex: 0 0 auto; display: flex; align-items: center; }
.card-delete, .card-retry { align-self: center; border: 0; border-radius: 8px; padding: 7px 9px; font-size: 12px; cursor: pointer; white-space: nowrap; }
.card-delete { color: #EF4444; background: #FEF2F2; }
.card-retry { color: #C15F18; background: #FFF3E6; }
.card-retry:disabled { cursor: wait; opacity: .65; }
.card-retry-area { grid-column: 1 / -1; display: flex; align-items: center; justify-content: flex-end; gap: 10px; min-width: 0; }
.card-retry-error { min-width: 0; overflow: hidden; color: #DC2626; font-size: 11px; line-height: 1.35; text-overflow: ellipsis; white-space: nowrap; }
.card-empty { padding: 42px 16px; border: 1px dashed #F1D9C3; border-radius: 16px; color: #9398AE; background: #fff; font-size: 14px; text-align: center; }
.qa-card { align-items: stretch; text-align: left; }.qa-card.qa-disabled { opacity: .58; }
.qa-card .card-content,
@@ -617,8 +638,11 @@ onUnmounted(stopDocumentPolling)
@media (max-width: 520px) {
.knowledge-panel { padding: 0 12px; }
.knowledge-card { display: grid; grid-template-columns: 42px minmax(0, 1fr); align-items: start; gap: 10px; padding: 13px; }
.document-card { grid-template-columns: 42px minmax(0, 1fr) auto; }
.card-content { grid-column: 2; }
.card-delete { grid-column: 2; justify-self: end; margin-top: -2px; }
.card-actions { grid-column: 3; grid-row: 1; }
.card-delete { justify-self: end; margin-top: -2px; }
.card-retry-area { grid-column: 1 / -1; }
.qa-card { display: block; }
.qa-card .card-content { width: 100%; grid-column: 1; }
.card-title-row { align-items: flex-start; flex-wrap: wrap; gap: 5px 7px; }