fix(avatar): prevent BOXIM polling starvation
This commit is contained in:
stefanfeng
@@ -25,7 +25,8 @@ logger = logging.getLogger(__name__)
|
||||
ACTIVE_TASK_STATUSES = ("pending", "generating", "ready", "sending")
|
||||
GENERATABLE_TASK_STATUSES = ("pending",)
|
||||
MAX_PROMPT_LENGTH = 4000
|
||||
MAX_STALE_SECONDS = 120
|
||||
DEFAULT_MAX_MESSAGE_AGE_SECONDS = 600
|
||||
MAX_SEND_OVERDUE_SECONDS = 120
|
||||
STUCK_LOCK_SECONDS = 90
|
||||
TAKEOVER_PERMISSION = "takeover"
|
||||
TAKEOVER_DELAY_KEY = "takeoverReplyDelaySeconds"
|
||||
@@ -115,11 +116,15 @@ class TakeoverService:
|
||||
boxim_client: BoxIMClient,
|
||||
*,
|
||||
reply_delay_seconds: int | None = None,
|
||||
poll_concurrency: int = 8,
|
||||
max_message_age_seconds: int = DEFAULT_MAX_MESSAGE_AGE_SECONDS,
|
||||
now: Callable[[], datetime] = _utcnow,
|
||||
):
|
||||
self.session_factory = session_factory
|
||||
self.boxim = boxim_client
|
||||
self.reply_delay_seconds = reply_delay_seconds
|
||||
self.poll_concurrency = max(1, min(int(poll_concurrency), 64))
|
||||
self.max_message_age_seconds = max(60, int(max_message_age_seconds))
|
||||
self.now = now
|
||||
self._sessions: dict[str, dict] = {}
|
||||
self._poll_lock = asyncio.Lock()
|
||||
@@ -138,8 +143,48 @@ class TakeoverService:
|
||||
self._recover_stuck_tasks()
|
||||
avatar_ids = self._enabled_avatar_ids()
|
||||
self._cancel_disabled_tasks(set(avatar_ids))
|
||||
for avatar_id in avatar_ids:
|
||||
await self._sync_avatar(avatar_id)
|
||||
self._ensure_takeover_cursors(avatar_ids)
|
||||
semaphore = asyncio.Semaphore(self.poll_concurrency)
|
||||
|
||||
async def sync(avatar_id: str):
|
||||
async with semaphore:
|
||||
return await self._sync_avatar(avatar_id)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(sync(avatar_id) for avatar_id in avatar_ids),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for avatar_id, result in zip(avatar_ids, results):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning("BOXIM poll crashed for avatar %s: %s", avatar_id, result)
|
||||
|
||||
def _ensure_takeover_cursors(self, avatar_ids: list[str]):
|
||||
"""Create durable cursors before concurrent network polling starts."""
|
||||
if not avatar_ids:
|
||||
return
|
||||
db = self.session_factory()
|
||||
try:
|
||||
existing = {
|
||||
row[0]
|
||||
for row in db.query(TakeoverCursor.avatar_id)
|
||||
.filter(TakeoverCursor.avatar_id.in_(avatar_ids))
|
||||
.all()
|
||||
}
|
||||
avatars = (
|
||||
db.query(Avatar.id, Avatar.owner_id)
|
||||
.filter(
|
||||
Avatar.id.in_(
|
||||
[avatar_id for avatar_id in avatar_ids if avatar_id not in existing]
|
||||
)
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for avatar_id, owner_id in avatars:
|
||||
db.add(TakeoverCursor(avatar_id=avatar_id, owner_id=owner_id))
|
||||
if avatars:
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
async def process_reply_tasks(self):
|
||||
"""Generate and send replies independently from BOXIM's long poll."""
|
||||
@@ -290,7 +335,10 @@ class TakeoverService:
|
||||
if not cursor:
|
||||
cursor = TakeoverCursor(avatar_id=avatar.id, owner_id=avatar.owner_id)
|
||||
db.add(cursor)
|
||||
db.flush()
|
||||
db.commit()
|
||||
else:
|
||||
# Release SQLite's read transaction before the long network poll.
|
||||
db.commit()
|
||||
if not user or not user.huihui_token:
|
||||
self._record_connection_failure(
|
||||
db,
|
||||
@@ -452,16 +500,37 @@ class TakeoverService:
|
||||
return
|
||||
if not schedule_reply or event.message_type != 0 or not event.content.strip():
|
||||
return
|
||||
if (now - send_time).total_seconds() > MAX_STALE_SECONDS:
|
||||
if (now - send_time).total_seconds() > self.max_message_age_seconds:
|
||||
logger.info(
|
||||
"Ignored stale BOXIM message %s for avatar %s (age=%ss)",
|
||||
message_id,
|
||||
avatar.id,
|
||||
int((now - send_time).total_seconds()),
|
||||
)
|
||||
return
|
||||
if is_avatar:
|
||||
self._cancel_conversation(db, avatar.owner_id, peer_id, "peer_avatar_message")
|
||||
logger.info(
|
||||
"Skipped BOXIM reply for avatar %s message %s: peer_avatar_message",
|
||||
avatar.id,
|
||||
message_id,
|
||||
)
|
||||
return
|
||||
if self._human_pause_active(db, avatar.owner_id, peer_id, now):
|
||||
self._cancel_conversation(db, avatar.owner_id, peer_id, "owner_active")
|
||||
logger.info(
|
||||
"Skipped BOXIM reply for avatar %s message %s: owner_active",
|
||||
avatar.id,
|
||||
message_id,
|
||||
)
|
||||
return
|
||||
if self._conversation_rate_limited(db, avatar.owner_id, peer_id, now):
|
||||
self._cancel_conversation(db, avatar.owner_id, peer_id, "rate_limited")
|
||||
logger.info(
|
||||
"Skipped BOXIM reply for avatar %s message %s: rate_limited",
|
||||
avatar.id,
|
||||
message_id,
|
||||
)
|
||||
return
|
||||
self._schedule_reply(db, avatar, event)
|
||||
|
||||
@@ -540,8 +609,10 @@ class TakeoverService:
|
||||
prompt_parts.append(event.content.strip())
|
||||
source_ids.append(event.boxim_message_id)
|
||||
prompt = "\n".join(part for part in prompt_parts if part).strip()[-MAX_PROMPT_LENGTH:]
|
||||
due_at = event.send_time + timedelta(
|
||||
seconds=_configured_reply_delay(avatar, self.reply_delay_seconds)
|
||||
due_at = max(
|
||||
event.send_time
|
||||
+ timedelta(seconds=_configured_reply_delay(avatar, self.reply_delay_seconds)),
|
||||
self.now(),
|
||||
)
|
||||
task_id = secrets.token_hex(16)
|
||||
local_id = _avatar_local_id(avatar.owner_id, event.boxim_message_id)
|
||||
@@ -701,7 +772,7 @@ class TakeoverService:
|
||||
task.cancel_reason = "takeover_disabled"
|
||||
db.commit()
|
||||
return False
|
||||
if (self.now() - task.scheduled_at).total_seconds() > MAX_STALE_SECONDS:
|
||||
if (self.now() - task.scheduled_at).total_seconds() > MAX_SEND_OVERDUE_SECONDS:
|
||||
task.status = "cancelled"
|
||||
task.cancel_reason = "stale_reply"
|
||||
db.commit()
|
||||
|
||||
Reference in New Issue
Block a user