Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 36 additions & 25 deletions src/agent_env/task_step/task_steps/always_on.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@
- The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report
back); the full timeline goes to the object store, at the summary's ``object_url``.
- An always_on_agent_verifier (``verifiers/always_on_agent_verifier.py``) grades the step's prompt from the user's chat
and the run's context, the run wake by wake among it (``render_for_judge``); a plain rubrics_verifier on it is
refused when the task loads.
and the run's context, the run wake by wake among it (``render_for_judge``), and, with use_trajectory, every wake's
tool calls staged as files; a plain rubrics_verifier on it is refused when the task loads.
- The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it,
else on its first wake; its session is the step's ``context_id`` when set, as prompt_agent's is, else the run's own.
- When the step ends it puts back the gateway settings it changed for the run (the user chat's roles and stay_quiet,
Expand Down Expand Up @@ -179,10 +179,6 @@
# wrote them, so checks that read its reply times (user_reads_at) grade a run on this step.
HOOK_RECORDS_KEY = "post_turn_hooks"
HOOK_NAME = "always_on"
CHANGE_SUMMARY_CHARS = 200
# The app changes one read of an env's changelog keeps in the timeline; the rest are counted (counts.app_changes counts
# them all, by who made them).
CHANGES_KEPT_PER_READ = 200
TIMELINE_TAIL = 20
# With no end_time, the run ends once this many clock jumps in a row have been to repeating triggers' marks alone (see
# the loop's docstring): the world can go on ticking, but nothing in it has woken the agent.
Expand Down Expand Up @@ -1410,9 +1406,8 @@ async def _read_changes(self, phase: str) -> None:
self.change_cursors[env_id] = (env, cursor)
if rows:
changes = [self._change(env_id, row, phase) for row in rows]
more = len(changes) - CHANGES_KEPT_PER_READ
self.log("app_changes", wake=self.wakes, phase=phase, env_id=env_id, count=len(changes),
changes=changes[:CHANGES_KEPT_PER_READ], **({"not_kept": more} if more > 0 else {}))
changes=changes)

def _change(self, env_id: str, row: dict, phase: str) -> dict:
actors = [{k: v for k, v in a.items() if k in ("kind", "role", "tool", "trigger_id", "request", "caused_by",
Expand All @@ -1421,11 +1416,8 @@ def _change(self, env_id: str, row: dict, phase: str) -> dict:
mine = any(a.get("role") == self.recipient and a.get("kind") in ("tool_call", "request") for a in actors)
by = "agent" if mine else "others" if actors else None
self.change_counts["by_agent" if mine else "by_others" if actors else "no_actor"] += 1
summary = row.get("summary")
if isinstance(summary, str) and len(summary) > CHANGE_SUMMARY_CHARS:
summary = summary[:CHANGE_SUMMARY_CHARS] + "…"
return {"env_id": env_id, "id": row.get("id"), "table": f"{row.get('schema_name')}.{row.get('table_name')}",
"op": row.get("operation"), "row_id": row.get("row_id"), "summary": summary,
"op": row.get("operation"), "row_id": row.get("row_id"), "summary": row.get("summary"),
"at": row.get("timestamp"), "by": by, "wake": self.wakes, "phase": phase, "actors": actors}

async def _take_messages(self) -> tuple[list[dict], list[dict]]:
Expand Down Expand Up @@ -1737,27 +1729,44 @@ def _append_prompt_response(self, final_text: str) -> None:
# What a judge reads of the run beyond the user's chat (always_on_agent_verifier, for an always_on step's prompt).
JUDGE_CHANGES_PER_WAKE = 10
JUDGE_WAKES = 300
JUDGE_SUMMARY_CHARS = 200 # of an app change's summary in the wake-by-wake text; the stored timeline has it whole
_WOKE_BY = {"user_request": "the user's request", "user_reply": "the user's reply", "user_says": "a user message",
"final": "the user's final line", "step:final": "the end of the job"}


def stored_timeline(record: dict) -> list[dict]:
"""The run's timeline from the object store (``record["object_url"]``), else the record's tail; raises when the
stored one can't be read."""
if not record.get("object_url"):
return record.get("timeline_tail") or []
url = record["object_url"]
return json.loads(get_config().get_object_store_at(url).get(url))["timeline"]


def woke_by_text(woke_by: list[str]) -> str:
"""What woke the agent, in words: "the user's reply, an alert (Gmail)"."""
return ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w)
for w in woke_by or []) or "nothing named"


def render_for_judge(record: dict) -> str:
"""The run wake by wake for a judge: each wake's pretend time, what woke the agent, how much it sent the user and
its tool calls, and the app changes made around it, the agent's own marked; then how the run ended and what was
still pending. From the stored timeline at ``record["object_url"]`` (else the record's tail)."""
timeline = record.get("timeline_tail") or []
if record.get("object_url"):
try:
url = record["object_url"]
timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"]
except Exception as e:
return f"## The run, wake by wake\n(The run's record could not be read: {type(e).__name__}: {e})"
still pending. From the stored timeline at ``record["object_url"]`` (else the record's tail). What it leaves out
(changes past JUDGE_CHANGES_PER_WAKE, wakes past JUDGE_WAKES, changes a run recorded before every change was
kept) it says."""
try:
timeline = stored_timeline(record)
except Exception as e:
return f"## The run, wake by wake\n(The run's record could not be read: {type(e).__name__}: {e})"
turns = {e["n"]: e for e in timeline if e["event"] == "agent_turn_end"}
nudges = {e["wake"]: e for e in timeline if e["event"] == "nudge"}
changes: dict[int, list[dict]] = {}
unrecorded: dict[int, int] = {}
for e in timeline:
if e["event"] == "app_changes":
changes.setdefault(e["wake"], []).extend(e.get("changes") or [])
unrecorded[e["wake"]] = unrecorded.get(e["wake"], 0) + (e.get("not_kept") or 0)
lines = ["## The run, wake by wake",
"The assistant ran only when something woke it. Each wake below gives its pretend time, what woke it, how "
"many messages it sent the user (their text is in the conversation) and its tool calls, then the app "
Expand All @@ -1767,15 +1776,14 @@ def render_for_judge(record: dict) -> str:
wakes = [e for e in timeline if e["event"] == "wake"]
for wake in wakes[:JUDGE_WAKES]:
n = wake["n"]
woke = ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w)
for w in wake.get("woke_by") or []) or "nothing named"
turn = turns.get(n) or {}
sent = (turn.get("messages_to_user") or 0) + (nudges.get(n, {}).get("messages_to_user") or 0)
calls = f" · {turn['tool_calls']} tool calls" if turn.get("tool_calls") is not None else ""
state = f" · its turn {turn['state']}" if turn.get("state") not in (None, "completed", "TaskState.completed") \
else ""
lines.append(f"- Wake {n} · {label_time(wake.get('virtual_time'))} · woken by {woke} · sent the user "
f"{sent} message{'s' if sent != 1 else ''}{calls}{state}")
lines.append(f"- Wake {n} · {label_time(wake.get('virtual_time'))} · woken by "
f"{woke_by_text(wake.get('woke_by'))} · sent the user {sent} message{'s' if sent != 1 else ''}"
f"{calls}{state}")
made = changes.get(n, [])
for c in made[:JUDGE_CHANGES_PER_WAKE]:
if c.get("by") == "agent":
Expand All @@ -1784,10 +1792,13 @@ def render_for_judge(record: dict) -> str:
who = f"{_actor(c)} changed ({c.get('phase')})"
else:
who = f"no known actor changed ({c.get('phase')})"
summary = f" ({c['summary']})" if c.get("summary") else ""
text = c.get("summary") or ""
summary = f" ({text[:JUDGE_SUMMARY_CHARS]}{'…' if len(text) > JUDGE_SUMMARY_CHARS else ''})" if text else ""
lines.append(f" - {who}: {c.get('op')} {c.get('table')} row {c.get('row_id')}{summary}")
if len(made) > JUDGE_CHANGES_PER_WAKE:
lines.append(f" - and {len(made) - JUDGE_CHANGES_PER_WAKE} more changes")
if unrecorded.get(n):
lines.append(f" - and {unrecorded[n]} changes the run didn't record")
if len(wakes) > JUDGE_WAKES:
lines.append(f"(and {len(wakes) - JUDGE_WAKES} more wakes)")
pending = {kind: v.get("count", 0) for kind, v in (record.get("pending_at_end") or {}).items()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ def compact_otel_trajectory(
events.extend(tool_events)
externalized_files.extend(result_files)
if trajectory_filter.include_response:
events.extend(_extract_final_response(spans))
events.extend(extract_final_response(spans))
events.sort(key=lambda e: e.get("timestamp", ""))

logger.info(
Expand Down Expand Up @@ -435,18 +435,10 @@ def _extract_tool_events(
result_file_counter = 0

for span in spans:
attrs = span.get("attributes", {})
if attrs.get("gen_ai.operation.name") != "execute_tool":
if not is_tool_span(span):
continue

tool_name = span.get("name", "")
prompt_raw = attrs.get("gen_ai.prompt", "{}")
prompt = parse_json_attr(prompt_raw)
tool_input = prompt.get("input", {}) if isinstance(prompt, dict) else prompt

completion_raw = attrs.get("gen_ai.completion", "{}")
completion = parse_json_attr(completion_raw)
tool_output = completion.get("output", completion) if isinstance(completion, dict) else completion
tool_name, tool_input, tool_output = tool_span_io(span)

if include_calls:
events.append({
Expand Down Expand Up @@ -480,7 +472,22 @@ def _extract_tool_events(
return events, externalized_files


def _extract_final_response(spans: list[dict]) -> list[dict]:
def is_tool_span(span: dict) -> bool:
return span.get("attributes", {}).get("gen_ai.operation.name") == "execute_tool"


def tool_span_io(span: dict) -> tuple[str, object, object]:
"""An ``execute_tool`` span's tool name, input and output, as its ``gen_ai.prompt`` / ``gen_ai.completion`` carry
them."""
attrs = span.get("attributes", {})
prompt = parse_json_attr(attrs.get("gen_ai.prompt", "{}"))
completion = parse_json_attr(attrs.get("gen_ai.completion", "{}"))
return (span.get("name", ""),
prompt.get("input", {}) if isinstance(prompt, dict) else prompt,
completion.get("output", completion) if isinstance(completion, dict) else completion)


def extract_final_response(spans: list[dict]) -> list[dict]:
"""Extract the final response from the conversation root span (gen_ai.operation.name == 'chain')."""
for span in spans:
attrs = span.get("attributes", {})
Expand Down
Loading
Loading