feat(avatar): understand BOXIM image messages

2026-09-01 15:35:41 +08:00
parent 3d999f9472
commit 0fc43908ae
8 changed files with 523 additions and 13 deletions
@@ -3,6 +3,7 @@
 import asyncio
 import hashlib
 import logging
+import os
 import re
 import secrets
 import time
@@ -13,12 +14,19 @@ from sqlalchemy.orm import Session

 from models import (
    Avatar,
+    ChatAttachment,
    TakeoverCursor,
    TakeoverMessage,
    TakeoverReplyTask,
    User,
 )
 from services.boxim_client import BoxIMClient, BoxIMError
+from services.boxim_image_service import (
+    BoxIMImageError,
+    download_boxim_image,
+    parse_boxim_image_url,
+)
+from services.vision_service import ImageValidationError

 logger = logging.getLogger(__name__)

@@ -37,6 +45,10 @@ HUMAN_PAUSE_SECONDS = 600
 RATE_LIMIT_WINDOW_SECONDS = 300
 RATE_LIMIT_MAX_REPLIES = 5
 AVATAR_LOCAL_ID_PREFIX = "880"
+BOXIM_TEXT_MESSAGE_TYPE = 0
+BOXIM_IMAGE_MESSAGE_TYPE = 1
+BOXIM_IMAGE_PROMPT = "请看看这张图片。"
+BOXIM_IMAGE_UNAVAILABLE_REPLY = "这张图片我暂时没看清，麻烦重新发送一张清晰的原图。"


 def _utcnow() -> datetime:
@@ -107,6 +119,14 @@ def _configured_reply_delay(avatar: Avatar, fallback: int | None = None) -> int:
    return delay


+def _event_prompt(event: TakeoverMessage) -> str:
+    if event.message_type == BOXIM_TEXT_MESSAGE_TYPE:
+        return event.content.strip()
+    if event.message_type == BOXIM_IMAGE_MESSAGE_TYPE:
+        return BOXIM_IMAGE_PROMPT
+    return ""
+
+
 class TakeoverService:
    """Poll BOXIM, honor the owner grace period, then generate and send one reply."""

@@ -498,8 +518,27 @@ class TakeoverService:
            if not is_avatar:
                self._cancel_conversation(db, avatar.owner_id, peer_id, "owner_replied")
            return
-        if not schedule_reply or event.message_type != 0 or not event.content.strip():
+        if not schedule_reply or event.message_type not in {
+            BOXIM_TEXT_MESSAGE_TYPE,
+            BOXIM_IMAGE_MESSAGE_TYPE,
+        }:
            return
+        if event.message_type == BOXIM_TEXT_MESSAGE_TYPE and not event.content.strip():
+            return
+        if event.message_type == BOXIM_IMAGE_MESSAGE_TYPE:
+            try:
+                parse_boxim_image_url(
+                    event.content,
+                    base_url=getattr(self.boxim, "im_base_url", ""),
+                )
+            except BoxIMImageError as exc:
+                logger.warning(
+                    "Ignored invalid BOXIM image message %s for avatar %s: %s",
+                    message_id,
+                    avatar.id,
+                    exc,
+                )
+                return
        if (now - send_time).total_seconds() > self.max_message_age_seconds:
            logger.info(
                "Ignored stale BOXIM message %s for avatar %s (age=%ss)",
@@ -606,7 +645,7 @@ class TakeoverService:
                task.status = "cancelled"
                task.cancel_reason = "newer_incoming_message"
                task.locked_at = None
-        prompt_parts.append(event.content.strip())
+        prompt_parts.append(_event_prompt(event))
        source_ids.append(event.boxim_message_id)
        prompt = "\n".join(part for part in prompt_parts if part).strip()[-MAX_PROMPT_LENGTH:]
        due_at = max(
@@ -665,6 +704,50 @@ class TakeoverService:
        results = await asyncio.gather(*(generate(task_id) for task_id in task_ids))
        return sum(bool(result) for result in results)

+    def _takeover_image_attachment(
+        self,
+        db: Session,
+        avatar: Avatar,
+        event: TakeoverMessage,
+    ) -> ChatAttachment:
+        now = self.now()
+        if event.attachment_id:
+            cached = db.get(ChatAttachment, event.attachment_id)
+            if cached and cached.status == "ready" and cached.expires_at > now:
+                cached.used_at = now
+                db.commit()
+                return cached
+
+        downloaded = download_boxim_image(
+            event.content,
+            base_url=getattr(
+                self.boxim,
+                "im_base_url",
+                os.getenv("BOXIM_API_BASE_URL", "https://im.99hui.com/api"),
+            ),
+        )
+        from routers.chat import _analyze_image_bytes
+
+        attachment = _analyze_image_bytes(
+            db,
+            avatar,
+            downloaded.content,
+            filename=downloaded.filename,
+            mime_type=downloaded.mime_type,
+            uploader_kind="boxim",
+        )
+        event.attachment_id = attachment.id
+        attachment.used_at = now
+        db.commit()
+        logger.info(
+            "BOXIM image analyzed message=%s attachment=%s avatar=%s category=%s",
+            event.boxim_message_id,
+            attachment.id,
+            avatar.id,
+            attachment.category,
+        )
+        return attachment
+
    def _generate_reply(self, task_id: str) -> bool:
        db = self.session_factory()
        try:
@@ -694,9 +777,36 @@ class TakeoverService:
                .limit(30)
                .all()
            )
+            source_events = {
+                event.boxim_message_id: event
+                for event in events
+                if event.boxim_message_id in excluded_ids
+            }
+            image_attachments = []
+            image_failed = False
+            for message_id in (task.source_message_ids or [])[-3:]:
+                event = source_events.get(message_id)
+                if not event or event.message_type != BOXIM_IMAGE_MESSAGE_TYPE:
+                    continue
+                try:
+                    image_attachments.append(
+                        self._takeover_image_attachment(db, avatar, event)
+                    )
+                except (BoxIMImageError, ImageValidationError) as exc:
+                    image_failed = True
+                    logger.warning(
+                        "BOXIM image unavailable message=%s avatar=%s: %s",
+                        event.boxim_message_id,
+                        avatar.id,
+                        exc,
+                    )
            history = []
            for event in reversed(events):
-                if event.boxim_message_id in excluded_ids or not event.content.strip():
+                if (
+                    event.boxim_message_id in excluded_ids
+                    or event.message_type != BOXIM_TEXT_MESSAGE_TYPE
+                    or not event.content.strip()
+                ):
                    continue
                if event.direction == "incoming" and event.is_avatar:
                    continue
@@ -708,10 +818,25 @@ class TakeoverService:
                )
            history = history[-10:]

-            from routers.chat import _resolve_reply
+            from routers.chat import _attachment_contexts, _resolve_reply

-            result = _resolve_reply(db, avatar, task.prompt, history, usage_source="takeover")
-            answer = _plain_text_reply(result.get("answer", ""))
+            image_contexts = _attachment_contexts(image_attachments)
+            has_source_text = any(
+                event.message_type == BOXIM_TEXT_MESSAGE_TYPE and event.content.strip()
+                for event in source_events.values()
+            )
+            if image_failed and not image_contexts and not has_source_text:
+                answer = BOXIM_IMAGE_UNAVAILABLE_REPLY
+            else:
+                result = _resolve_reply(
+                    db,
+                    avatar,
+                    task.prompt,
+                    history,
+                    usage_source="takeover",
+                    image_contexts=image_contexts,
+                )
+                answer = _plain_text_reply(result.get("answer", ""))
            db.refresh(task)
            if task.status != "generating":
                return False