From 6a4b35c49a4df2cbce3089115bdd783a58b03f66 Mon Sep 17 00:00:00 2001 From: stefanfeng Date: Wed, 2 Sep 2026 14:47:27 +0800 Subject: [PATCH] fix(avatar): ground replies in recognized images --- digital-avatar-app/backend/routers/chat.py | 100 +++++++++++++++++- .../backend/services/takeover_service.py | 31 +++++- .../backend/tests/test_chat_images.py | 42 ++++++++ .../backend/tests/test_takeover_service.py | 48 +++++++++ 4 files changed, 217 insertions(+), 4 deletions(-) diff --git a/digital-avatar-app/backend/routers/chat.py b/digital-avatar-app/backend/routers/chat.py index 69e9914..be5a7c1 100644 --- a/digital-avatar-app/backend/routers/chat.py +++ b/digital-avatar-app/backend/routers/chat.py @@ -47,6 +47,25 @@ QA_SEMANTIC_THRESHOLD = 0.72 QA_MATCH_MARGIN = 0.06 KNOWLEDGE_MIN_SCORE = float(os.getenv("KNOWLEDGE_MIN_SCORE", "0.42")) +_IMAGE_ACCESS_DENIAL_PATTERNS = ( + re.compile( + r"(?:我|目前|暂时|这里|本身|系统)?\s*(?:无法|不能|没法|不支持)\s*" + r"(?:直接)?\s*(?:查看|看到|看见|识别|读取|访问|打开|分析|理解)" + r"(?:\s*(?:或|、|/)\s*(?:查看|看到|看见|识别|读取|访问|打开|分析|理解))*\s*" + r"(?:你(?:发|提供|上传)的|这张|该|当前)?\s*(?:图片|图像|照片|影像|文件)" + ), + re.compile( + r"(?:我|这里|目前|暂时)?\s*(?:看不到|看不见|未看到|没有看到|没收到|未收到)\s*" + r"(?:你(?:发|提供|上传)的|这张|该|当前)?\s*(?:图片|图像|照片|影像)" + ), + re.compile( + r"\b(?:i\s+)?(?:can(?:not|'t)|am\s+unable\s+to)\s+(?:directly\s+)?" + r"(?:view|see|access|read|analy[sz]e|recogni[sz]e)\s+" + r"(?:the\s+|this\s+|your\s+)?(?:image|photo|picture|scan)\b", + re.IGNORECASE, + ), +) + _WRITING_SYSTEM_PATTERNS = { "han": re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]"), "latin": re.compile(r"[A-Za-z\u00c0-\u024f]"), @@ -178,6 +197,75 @@ def _image_retrieval_question(question: str, image_contexts: list[dict]) -> str: return "\n".join(part for part in parts if part).strip() +def _answer_denies_available_image(answer: str) -> bool: + """Reject only whole-image access denials, not uncertainty about one field.""" + value = re.sub(r"\s+", " ", answer or "").strip() + return any(pattern.search(value) for pattern in _IMAGE_ACCESS_DENIAL_PATTERNS) + + +def _compact_context_text(value: Any, limit: int) -> str: + lines = [re.sub(r"\s+", " ", line).strip() for line in str(value or "").splitlines()] + text = "\n".join(line for line in lines if line).strip() + return text[:limit].rstrip() + + +def _grounded_image_fallback(question: str, image_contexts: list[dict]) -> str: + """Build a safe answer from completed vision data when the chat model contradicts it.""" + summaries: list[str] = [] + facts: list[str] = [] + excerpts: list[str] = [] + warnings: list[str] = [] + for context in image_contexts: + summary = _compact_context_text(context.get("summary"), 500) + if summary: + summaries.append(summary) + structured = context.get("structuredData") or {} + if isinstance(structured, dict): + for fact in structured.get("key_facts") or []: + value = _compact_context_text(fact, 300) + if value: + facts.append(value) + extracted = _compact_context_text(context.get("extractedText"), 900) + if extracted: + excerpts.append(extracted) + warning = _compact_context_text(context.get("warning"), 300) + if warning: + warnings.append(warning) + + summaries = list(dict.fromkeys(summaries)) + facts = list(dict.fromkeys(facts))[:6] + excerpts = list(dict.fromkeys(excerpts)) + warnings = list(dict.fromkeys(warnings)) + writing_system = _dominant_writing_system(question) + + if writing_system == "latin": + parts = [] + if summaries: + parts.append("From the image, I can confirm: " + " ".join(summaries)) + if facts: + parts.append("Key details:\n" + "\n".join( + f"{index}. {fact}" for index, fact in enumerate(facts, 1) + )) + elif excerpts: + parts.append("Visible text:\n" + excerpts[0]) + if warnings: + parts.append("Please note: " + " ".join(warnings)) + return "\n".join(parts).strip() or "The image is available, but there is not enough clear detail to confirm more." + + parts = [] + if summaries: + parts.append("从这张图中可以确认：" + "；".join(summaries).rstrip("。；") + "。") + if facts: + parts.append("其中比较明确的信息有：\n" + "\n".join( + f"{index}. {fact}" for index, fact in enumerate(facts, 1) + )) + elif excerpts: + parts.append("图中可见的主要文字是：\n" + excerpts[0]) + if warnings: + parts.append("需要注意：" + "；".join(warnings).rstrip("。；") + "。") + return "\n".join(parts).strip() or "这张图已经看到了，但目前能确认的清晰信息比较有限。" + + def _run_billed_vision_call( db: Session, avatar: Avatar, @@ -553,8 +641,10 @@ def _build_prompt( if image_contexts: image_material = json.dumps(image_contexts, ensure_ascii=False, default=str) system += ( - "\n以下是当前会话图片经过视觉识别后得到的资料：\n" + "\n当前会话图片已经成功读取并完成内容识别，以下资料就是可直接使用的图片内容：\n" f"{image_material}" + "\n必须直接依据这些图片内容回答当前问题。禁止声称无法查看、看不到、未收到、无法识别、" + "无法读取或不能访问图片，也不要要求对方重新上传；只有资料明确标记读取失败时才可以请对方重发。" "\n图片资料可能包含 OCR 错字、模糊内容或用户尚未确认的信息，只能按可见内容谨慎表达。" "标准答题对中的事实优先级高于图片资料，知识库事实优先级高于模型推测；发生冲突时遵循更高优先级资料，" "并自然提醒对方核对原图。不得声称看到了图片中不存在的内容。" @@ -815,6 +905,14 @@ def _resolve_reply( except Exception as exc: release_reservation(db, reservation, str(exc)) raise + answer = str(answer or "").strip() + if image_contexts and _answer_denies_available_image(answer): + logger.warning( + "chat model contradicted ready image context avatar=%s source=%s", + avatar.id, + usage_source, + ) + answer = _grounded_image_fallback(question, image_contexts) result = { "answer": answer, "source": "qa" if matched else ( diff --git a/digital-avatar-app/backend/services/takeover_service.py b/digital-avatar-app/backend/services/takeover_service.py index 47c1600..7e6d51e 100644 --- a/digital-avatar-app/backend/services/takeover_service.py +++ b/digital-avatar-app/backend/services/takeover_service.py @@ -50,8 +50,15 @@ BOXIM_IMAGE_MESSAGE_TYPE = 1 BOXIM_IMAGE_PROMPT = "请看看这张图片。" BOXIM_IMAGE_UNAVAILABLE_REPLY = "这张图片我暂时没看清，麻烦重新发送一张清晰的原图。" IMAGE_CONTEXT_LOOKBACK_SECONDS = 1800 +IMAGE_REFERENCE_LOOKBACK_SECONDS = 172_800 MAX_RECENT_IMAGE_CONTEXTS = 3 +_IMAGE_REFERENCE_PATTERN = re.compile( + r"(?:图片|图像|照片|截图|这张图|刚才.{0,8}图|病例|病历|检查单|检验单|化验单|报告|影像|" + r"\b(?:image|photo|picture|screenshot|scan|report)\b)", + re.IGNORECASE, +) + def _utcnow() -> datetime: return datetime.utcnow() @@ -129,6 +136,10 @@ def _event_prompt(event: TakeoverMessage) -> str: return "" +def _references_recent_image(value: str) -> bool: + return bool(_IMAGE_REFERENCE_PATTERN.search(value or "")) + + class TakeoverService: """Poll BOXIM, honor the owner grace period, then generate and send one reply.""" @@ -693,8 +704,14 @@ class TakeoverService: event: TakeoverMessage, current_source_ids: list[str], ) -> list[TakeoverMessage]: - """Recover a recent image that an older deployment recorded without a task.""" - threshold = event.send_time - timedelta(seconds=IMAGE_CONTEXT_LOOKBACK_SECONDS) + """Recover missed images, or reuse a referenced image from the last two days.""" + references_image = _references_recent_image(event.content) + lookback_seconds = ( + IMAGE_REFERENCE_LOOKBACK_SECONDS + if references_image + else IMAGE_CONTEXT_LOOKBACK_SECONDS + ) + threshold = event.send_time - timedelta(seconds=lookback_seconds) candidates = ( db.query(TakeoverMessage) .filter( @@ -714,7 +731,15 @@ class TakeoverService: if not candidates: return [] - handled_ids = set(current_source_ids) + current_ids = set(current_source_ids) + if references_image: + return [ + image + for image in reversed(candidates) + if image.boxim_message_id not in current_ids + ] + + handled_ids = set(current_ids) task_sources = ( db.query(TakeoverReplyTask.source_message_ids) .filter( diff --git a/digital-avatar-app/backend/tests/test_chat_images.py b/digital-avatar-app/backend/tests/test_chat_images.py index 1fc73b1..cca4a43 100644 --- a/digital-avatar-app/backend/tests/test_chat_images.py +++ b/digital-avatar-app/backend/tests/test_chat_images.py @@ -12,6 +12,7 @@ from main import app from models import ChatAttachment from routers.chat import ( ChatIn, + _answer_denies_available_image, _attachment_contexts, _load_chat_attachments, _resolve_reply, @@ -274,6 +275,47 @@ def test_image_context_keeps_standard_answer_authoritative(): assert "标准答题对中的事实优先级高于图片资料" in system +def test_ready_image_context_never_returns_whole_image_access_denial(): + avatar = SimpleNamespace( + id="avatar-vision", + name="测试分身", + description="产品顾问", + config={}, + ) + model = Mock(return_value="抱歉，我无法查看或识别图片，请重新上传。") + result = _resolve_reply( + None, + avatar, + "请看看这张图片", + [], + qa_pairs=[], + search_fn=Mock(return_value=[]), + model_client=model, + image_contexts=[{ + "id": "attachment", + "filename": "report.jpg", + "category": "medical_document", + "summary": "一份耳鼻喉科门诊记录", + "extractedText": "主诉：咽痛三天", + "structuredData": {"key_facts": ["主诉为咽痛三天"]}, + "warning": "请核对原始资料", + }], + ) + + assert result["source"] == "vision" + assert "一份耳鼻喉科门诊记录" in result["answer"] + assert "主诉为咽痛三天" in result["answer"] + assert "无法查看" not in result["answer"] + system = model.call_args.kwargs["messages"][0]["content"] + assert "当前会话图片已经成功读取" in system + assert "禁止声称无法查看" in system + + +def test_image_denial_detector_allows_uncertain_field_in_ready_image(): + assert _answer_denies_available_image("我无法查看这张图片") is True + assert _answer_denies_available_image("图片中患者姓名无法辨认，主诉为咽痛三天。") is False + + def test_attachment_context_does_not_expose_internal_fields(): row = SimpleNamespace( id="attachment", diff --git a/digital-avatar-app/backend/tests/test_takeover_service.py b/digital-avatar-app/backend/tests/test_takeover_service.py index ed17dcf..476e5ed 100644 --- a/digital-avatar-app/backend/tests/test_takeover_service.py +++ b/digital-avatar-app/backend/tests/test_takeover_service.py @@ -363,6 +363,54 @@ async def test_followup_text_recovers_recent_image_recorded_without_task(service db.close() +@pytest.mark.asyncio +async def test_explicit_followup_reuses_handled_image_within_two_days(service_context): + session_factory, service, boxim, clock = service_context + await service.poll_and_process_messages() + image_message = { + "id": 116, + "localId": 116, + "sendId": 200, + "recvId": 100, + "sendTime": clock.millis(), + "type": 1, + "content": json.dumps({"originUrl": "https://cdn.example/handled-case.png"}), + } + boxim.messages.append(image_message) + await service.poll_messages() + + db = session_factory() + try: + image_task = db.query(TakeoverReplyTask).filter_by(trigger_message_id="116").one() + image_task.status = "sent" + image_task.sent_at = clock.now() + db.commit() + finally: + db.close() + + clock.advance(47 * 60 * 60) + boxim.messages.append( + { + "id": 117, + "localId": 117, + "sendId": 200, + "recvId": 100, + "sendTime": clock.millis(), + "type": 0, + "content": "重新看一下刚才那张病例图片", + } + ) + await service.poll_messages() + + db = session_factory() + try: + task = db.query(TakeoverReplyTask).filter_by(trigger_message_id="117").one() + assert task.source_message_ids == ["116", "117"] + assert task.prompt == "请看看这张图片。\n重新看一下刚才那张病例图片" + finally: + db.close() + + @pytest.mark.asyncio async def test_invalid_image_message_is_recorded_but_not_scheduled(service_context): session_factory, service, boxim, clock = service_context -- 2.54.0