feat(avatar): understand BOXIM image messages

2026-09-01 15:35:41 +08:00
parent 3d999f9472
commit 0fc43908ae
8 changed files with 523 additions and 13 deletions
@@ -1,6 +1,7 @@
 """End-to-end service tests for BOXIM takeover timing and human priority."""

 import asyncio
+import json
 from datetime import datetime, timedelta, timezone
 from threading import Barrier
 from unittest.mock import AsyncMock, patch
@@ -10,8 +11,9 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker

 from database import Base
-from models import Avatar, TakeoverCursor, TakeoverMessage, TakeoverReplyTask, User
+from models import Avatar, ChatAttachment, TakeoverCursor, TakeoverMessage, TakeoverReplyTask, User
 from services.boxim_client import BoxIMError
+from services.boxim_image_service import DownloadedBoxIMImage
 from services.takeover_service import (
    AVATAR_LOCAL_ID_PREFIX,
    TakeoverService,
@@ -171,6 +173,121 @@ async def test_incoming_message_is_prepared_then_sent_at_three_seconds(service_c
        db.close()


+@pytest.mark.asyncio
+async def test_incoming_image_is_analyzed_and_used_in_takeover_reply(service_context):
+    session_factory, service, boxim, clock = service_context
+    await service.poll_and_process_messages()
+    boxim.messages.append(
+        {
+            "id": 111,
+            "localId": 111,
+            "sendId": 200,
+            "recvId": 100,
+            "sendTime": clock.millis(),
+            "type": 1,
+            "content": json.dumps(
+                {
+                    "originUrl": "https://cdn.example/case.png",
+                    "thumbUrl": "https://cdn.example/case-thumb.png",
+                }
+            ),
+        }
+    )
+
+    await service.poll_and_process_messages()
+    db = session_factory()
+    try:
+        scheduled = db.query(TakeoverReplyTask).filter_by(trigger_message_id="111").one()
+        assert scheduled.status == "pending"
+        assert scheduled.prompt == "请看看这张图片。"
+    finally:
+        db.close()
+    clock.advance(3)
+
+    def analyze(db, avatar, content, **kwargs):
+        assert content == b"image-content"
+        attachment = ChatAttachment(
+            avatar_id=avatar.id,
+            uploader_kind=kwargs["uploader_kind"],
+            filename=kwargs["filename"],
+            mime_type="image/jpeg",
+            file_size=len(content),
+            status="ready",
+            category="medical_document",
+            summary="一张门诊病例",
+            extracted_text="主诉：咳嗽三天",
+            structured_data={"medical": {"chief_complaint": "咳嗽三天"}},
+            warning="请核对原始资料",
+            expires_at=clock.now() + timedelta(hours=24),
+        )
+        db.add(attachment)
+        db.commit()
+        db.refresh(attachment)
+        return attachment
+
+    downloaded = DownloadedBoxIMImage(
+        content=b"image-content",
+        filename="case.png",
+        mime_type="image/png",
+        source_url="https://cdn.example/case.png",
+    )
+    with (
+        patch("services.takeover_service.download_boxim_image", return_value=downloaded),
+        patch("routers.chat._analyze_image_bytes", side_effect=analyze) as analyzer,
+        patch("routers.chat._resolve_reply", return_value={"answer": "这份资料里写的是咳嗽三天。"}) as resolver,
+    ):
+        await service.poll_and_process_messages()
+
+    analyzer.assert_called_once()
+    assert resolver.call_args.args[2] == "请看看这张图片。"
+    image_contexts = resolver.call_args.kwargs["image_contexts"]
+    assert image_contexts[0]["summary"] == "一张门诊病例"
+    assert image_contexts[0]["extractedText"] == "主诉：咳嗽三天"
+    assert [item["content"] for item in boxim.sent] == ["这份资料里写的是咳嗽三天。"]
+
+    db = session_factory()
+    try:
+        event = db.query(TakeoverMessage).filter_by(boxim_message_id="111").one()
+        task = db.query(TakeoverReplyTask).filter_by(trigger_message_id="111").one()
+        assert event.attachment_id
+        assert db.get(ChatAttachment, event.attachment_id).uploader_kind == "boxim"
+        assert task.status == "sent"
+        with patch(
+            "services.takeover_service.download_boxim_image",
+            side_effect=AssertionError("cached image must not be downloaded again"),
+        ):
+            cached = service._takeover_image_attachment(db, db.get(Avatar, "avatar-1"), event)
+        assert cached.id == event.attachment_id
+    finally:
+        db.close()
+
+
+@pytest.mark.asyncio
+async def test_invalid_image_message_is_recorded_but_not_scheduled(service_context):
+    session_factory, service, boxim, clock = service_context
+    await service.poll_and_process_messages()
+    boxim.messages.append(
+        {
+            "id": 112,
+            "localId": 112,
+            "sendId": 200,
+            "recvId": 100,
+            "sendTime": clock.millis(),
+            "type": 1,
+            "content": json.dumps({"width": 100, "height": 100}),
+        }
+    )
+
+    await service.poll_and_process_messages()
+
+    db = session_factory()
+    try:
+        assert db.query(TakeoverMessage).filter_by(boxim_message_id="112").one()
+        assert db.query(TakeoverReplyTask).count() == 0
+    finally:
+        db.close()
+
+
@pytest.mark.asyncio
 async def test_default_reply_delay_is_three_minutes(service_context):
    session_factory, service, boxim, clock = service_context