| 1 | """Post-run evaluation for background tasks (heartbeat & cron). |
| 2 | |
| 3 | After the agent executes a background task, this module makes a lightweight |
| 4 | LLM call to decide whether the result warrants notifying the user. |
| 5 | """ |
| 6 | |
| 7 | from __future__ import annotations |
| 8 | |
| 9 | from typing import TYPE_CHECKING |
| 10 | |
| 11 | from loguru import logger |
| 12 | |
| 13 | from nanobot.utils.prompt_templates import render_template |
| 14 | |
| 15 | if TYPE_CHECKING: |
| 16 | from nanobot.providers.base import LLMProvider |
| 17 | |
| 18 | _EVALUATE_TOOL = [ |
| 19 | { |
| 20 | "type": "function", |
| 21 | "function": { |
| 22 | "name": "evaluate_notification", |
| 23 | "description": "Decide whether the user should be notified about this background task result.", |
| 24 | "parameters": { |
| 25 | "type": "object", |
| 26 | "properties": { |
| 27 | "should_notify": { |
| 28 | "type": "boolean", |
| 29 | "description": "true = result contains actionable/important info the user should see; false = routine or empty, safe to suppress", |
| 30 | }, |
| 31 | "reason": { |
| 32 | "type": "string", |
| 33 | "description": "One-sentence reason for the decision", |
| 34 | }, |
| 35 | }, |
| 36 | "required": ["should_notify"], |
| 37 | }, |
| 38 | }, |
| 39 | } |
| 40 | ] |
| 41 | |
| 42 | async def evaluate_response( |
| 43 | response: str, |
| 44 | task_context: str, |
| 45 | provider: LLMProvider, |
| 46 | model: str, |
| 47 | ) -> bool: |
| 48 | """Decide whether a background-task result should be delivered to the user. |
| 49 | |
| 50 | Uses a lightweight tool-call LLM request (same pattern as heartbeat |
| 51 | ``_decide()``). Falls back to ``True`` (notify) on any failure so |
| 52 | that important messages are never silently dropped. |
| 53 | """ |
| 54 | try: |
| 55 | llm_response = await provider.chat_with_retry( |
| 56 | messages=[ |
| 57 | {"role": "system", "content": render_template("agent/evaluator.md", part="system")}, |
| 58 | {"role": "user", "content": render_template( |
| 59 | "agent/evaluator.md", |
| 60 | part="user", |
| 61 | task_context=task_context, |
| 62 | response=response, |
| 63 | )}, |
| 64 | ], |
| 65 | tools=_EVALUATE_TOOL, |
| 66 | model=model, |
| 67 | max_tokens=256, |
| 68 | temperature=0.0, |
| 69 | ) |
| 70 | |
| 71 | if not llm_response.should_execute_tools: |
| 72 | if llm_response.has_tool_calls: |
| 73 | logger.warning( |
| 74 | "evaluate_response: ignoring tool calls under finish_reason='{}', defaulting to notify", |
| 75 | llm_response.finish_reason, |
| 76 | ) |
| 77 | else: |
| 78 | logger.warning("evaluate_response: no tool call returned, defaulting to notify") |
| 79 | return True |
| 80 | |
| 81 | args = llm_response.tool_calls[0].arguments |
| 82 | should_notify = args.get("should_notify", True) |
| 83 | reason = args.get("reason", "") |
| 84 | logger.info("evaluate_response: should_notify={}, reason={}", should_notify, reason) |
| 85 | return bool(should_notify) |
| 86 | |
| 87 | except Exception: |
| 88 | logger.exception("evaluate_response failed, defaulting to notify") |
| 89 | return True |
| 90 |