From c70992e51ae022a66b1fceabb258d631e58da792 Mon Sep 17 00:00:00 2001 From: "wanhae.lee" Date: Sun, 5 Jul 2026 00:41:50 +0900 Subject: [PATCH] . --- report.md | 275 +++++++++++++++++++++++++++++++++++++-- test_gemma4_scenarios.py | 57 ++++---- 2 files changed, 292 insertions(+), 40 deletions(-) diff --git a/report.md b/report.md index 6c9b057..e60ebe1 100644 --- a/report.md +++ b/report.md @@ -46,81 +46,328 @@ temperature=1.5, top_p=1.0, max_new_tokens=640, reasoning=False - `well_formed` — every tool call block is complete and parseable (sglang parser port) - `valid_name` — every called tool exists - `schema_valid` — parsed arguments validate against the tool's JSON schema -- `stops_at_boundary` — generation did not run past into <|tool_response> +- `stops_at_boundary` — generation did not run past into <|tool_response>. Per the Gemma 4 prompt-formatting spec, the model is only responsible for the call up to ; <|tool_response> is appended by the application with the real tool result, and is registered as an additional stop sequence purely as a backstop. This harness configures no such stop, so unconstrained runs free-run past the boundary into engine-owned territory, while required-mode grammar ends the call cleanly at an accept state. - `thought_ok` — <|channel>thought opened and closed before the first tool call -## Failure examples +## Sample outputs (raw) -**A. complex-schema / unconstrained #0** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +### A. complex-schema + +**A. complex-schema / constrained #0** — PASS ✅ + +``` +<|tool_call>call:create_calendar_event{title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,duration_minutes:90,priority:<|"|>high<|"|>,attendees:[{name:<|"|>Alice Kim<|"|>,email:<|"|>alice.kim@example.com<|"|>},{name:<|"|>Bob Lee<|"|>,email:<|"|>bob.lee@example.com<|"|>},{name:<|"|>Chris Park<|"|>,email:<|"|>chris.park@example.com<|"|>}],location:{ room:<|"|>Jupiter<|"|>,floor:7},send_invites:true} +``` + +**A. complex-schema / constrained #1** — PASS ✅ + +``` +<|tool_call>call:create_calendar_event{ + title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>, + start:<|"|>2026-07-10T14:00:00+09:00<|"|>, + duration_minutes: 90, + priority:<|"|>high<|"|>, + attendees: [ + { + name:<|"|>Alice Kim<|"|>, + email:<|"|>alice.kim@example.com<|"|> + }, + { + name:<|"|>Bob Lee<|"|>, + email:<|"|>bob.lee@example.com<|"|> + }, + { + name:<|"|>Chris Park<|"|>, + email:<|"|>chris.park@example.com<|"|> + } + ], + location: { + room:<|"|>Jupiter<|"|>, + floor: 7 + }, + send_invites: true +} +``` + +**A. complex-schema / constrained #2** — PASS ✅ + +``` +<|tool_call>call:create_calendar_event{ + title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>, + start:<|"|>2026-07-10T14:00:00+08:00<|"|>, + duration_minutes: 90, + priority:<|"|>high<|"|>, + attendees: [ + { + name:<|"|>Alice Kim<|"|>, + email:<|"|>alice.kim@example.com<|"|> + }, + { + name:<|"|>Bob Lee<|"|>, + email:<|"|>bob.lee@example.com<|"|> + }, + { + name:<|"|>Chris Park<|"|>, + email:<|"|>chris.park@example.com<|"|> + } + ], + location: { + room:<|"|>Jupiter<|"|>, + floor: 7 + }, + send_invites: true +} +``` + +**A. complex-schema / constrained #3** — PASS ✅ + +``` +<|tool_call>call:create_calendar_event{title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,duration_minutes:90,priority:<|"|>high<|"|>,attendees:[{name:<|"|>Alice Kim<|"|>,email:<|"|>alice.kim@example.com<|"|>},{name:<|"|>Bob Lee<|"|>,email:<|"|>bob.lee@example.com<|"|>},{name:<|"|>Chris Park<|"|>,email:<|"|>chris.park@example.com<|"|>}],location:{room:<|"|>Jupiter<|"|>,floor:7},send_invites:true} +``` + +**A. complex-schema / unconstrained #0** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_calendar_event{attendees:[{email:<|"|>alice.kim@example.com<|"|>,name:<|"|>Alice Kim<|"|>},{email:<|"|>bob.lee@example.com<|"|>,name:<|"|>Bob Lee<|"|>},{email:<|"|>chris.park@example.com<|"|>,name:<|"|>Chris Park<|"|>}],duration_minutes:90,location:{floor:7,room:<|"|>Jupiter<|"|>},priority:<|"|>high<|"|>,send_invites:true,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>}<|tool_response> ``` -**A. complex-schema / unconstrained #1** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**A. complex-schema / unconstrained #1** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_calendar_event{attendees:[{email:<|"|>alice.kim@example.com<|"|>,name:<|"|>Alice Kim<|"|>},{email:<|"|>bob.lee@example.com<|"|>,name:<|"|>Bob Lee<|"|>},{email:<|"|>chris.park@example.com<|"|>,name:<|"|>Chris Park<|"|>}],duration_minutes:90,location:{floor:7,room:<|"|>Jupiter<|"|>},priority:<|"|>high<|"|>,send_invites:true,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>}<|tool_response> ``` -**A. complex-schema / unconstrained #2** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**A. complex-schema / unconstrained #2** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_calendar_event{attendees:[{email:<|"|>alice.kim@example.com<|"|>,name:<|"|>Alice Kim<|"|>},{email:<|"|>bob.lee@example.com<|"|>,name:<|"|>Bob Lee<|"|>},{email:<|"|>chris.park@example.com<|"|>,name:<|"|>Chris Park<|"|>}],duration_minutes:90,location:{floor:7,room:<|"|>Jupiter<|"|>},priority:<|"|>high<|"|>,send_invites:true,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>}<|tool_response> ``` -**A. complex-schema / unconstrained #3** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**A. complex-schema / unconstrained #3** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_calendar_event{attendees:[{email:<|"|>alice.kim@example.com<|"|>,name:<|"|>Alice Kim<|"|>},{email:<|"|>bob.lee@example.com<|"|>,name:<|"|>Bob Lee<|"|>},{email:<|"|>chris.park@example.com<|"|>,name:<|"|>Chris Park<|"|>}],duration_minutes:90,location:{floor:7,room:<|"|>Jupiter<|"|>},priority:<|"|>high<|"|>,send_invites:true,start:<|"|>2026-07-10T14:00:00+09:00<|"|>,title:<|"|>Q3 Roadmap Review ("final" draft)<|"|>}<|tool_response> ``` -**B. multiturn-thinking / unconstrained #0** — failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section +### B. multiturn-thinking + +**B. multiturn-thinking / constrained #0** — PASS ✅ + +``` +<|channel>thought +The user wants to compare the weather in Seoul and Busan in Celsius, one city at a time. +After getting both temperatures, the user specifically requested to convert Seoul's temperature to Fahrenheit using the `convert_temperature` tool before providing the final answer. + +1. **Seoul Weather (Celsius):** Already retrieved: 3°C. +2. **Busan Weather (Celsius):** Already retrieved: 8°C. +3. **Convert Seoul's temperature to Fahrenheit:** I need to call `convert_temperature` with `value=3`, `from_unit="celsius"`, and `to_unit="fahrenheit"`. + +After this conversion, I will present the comparison and the converted temperature. + +**Plan:** +1. Use the result from the previous calls (Seoul: 3°C, Busan: 8°C). +2. Call `convert_temperature` for Seoul (3°C to Fahrenheit). +3. Synthesize the final answer.<|tool_call>call:convert_temperature{ + value:3, + from_unit:<|"|>celsiu +``` + +**B. multiturn-thinking / constrained #1** — PASS ✅ + +``` +<|channel>thought +The user wants me to perform three main tasks: +1. Get the current weather in Seoul in Celsius. (Done) +2. Get the current weather in Busan in Celsius. (Done) +3. Convert Seoul's temperature from Celsius to Fahrenheit using the `convert_temperature` tool. +4. Compare the weather for both cities and present the final answer, including the conversion result. + +From the previous steps, I have: +* Seoul weather: 3°C and sunny. +* Busan weather: 8°C and cloudy. + +Now I need to call the `convert_temperature` tool for Seoul's temperature (3°C to Fahrenheit). + +After that, I will compile all the information to answer the user's request.<|tool_call>call:convert_temperature{ + value:3, + from_unit:<|"|>celsius<|"|>, + to_unit:<|"|>fahrenheit<|"|> +} +``` + +**B. multiturn-thinking / constrained #2** — PASS ✅ + +``` +<|channel>thought +The user wants to compare the current weather in Seoul and Busan in Celsius, one city at a time. +After getting both, the user wants to convert Seoul's temperature to Fahrenheit using the `convert_temperature` tool before providing the final answer. + +1. **Get Seoul's weather in Celsius:** This has been done. The result is: `{"temp_c": 3, "condition": "sunny", "humidity": 41}`. +2. **Get Busan's weather in Celsius:** This has been done. The result is: `{"temp_c": 8, "condition": "cloudy", "humidity": 63}`. +3. **Convert Seoul's temperature to Fahrenheit:** I need to use the `convert_temperature` tool with the Seoul temperature (3°C) from Celsius to Fahrenheit. + +Now I will execute the conversion call. +After that, I will compile all the information to answer the user's request.<|tool_call>call:convert_temperature{value:3,from_unit:<|"|>celsius<|"|>,to_unit:<|"|>f +``` + +**B. multiturn-thinking / constrained #3** — PASS ✅ + +``` +<|channel>thought +The user wants to: +1. Get the current weather in Seoul in Celsius. (Done) +2. Get the current weather in Busan in Celsius. (Done) +3. Convert Seoul's temperature from Celsius to Fahrenheit using the `convert_temperature` tool. (Pending) +4. Compare the weather, presenting the results one city at a time, and including the conversion. + +I have the following information: +* Seoul weather: 3°C, sunny, 41% humidity. +* Busan weather: 8°C, cloudy, 63% humidity. + +Now I need to call `convert_temperature` for Seoul's temperature (3°C to Fahrenheit). + +After that, I will compile the final answer.<|tool_call>call:convert_temperature{value:3,from_unit:<|"|>celsius<|"|>,to_unit:<|"|>fahrenheit<|"|>} +``` + +**B. multiturn-thinking / unconstrained #0** — FAIL ❌ (failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section) ``` <|tool_call>call:convert_temperature{from_unit:<|"|>celsius<|"|>,to_unit:<|"|>fahrenheit<|"|>,value:3}<|tool_response> ``` -**B. multiturn-thinking / unconstrained #1** — failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section +**B. multiturn-thinking / unconstrained #1** — FAIL ❌ (failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section) ``` <|tool_call>call:convert_temperature{from_unit:<|"|>celsius<|"|>,to_unit:<|"|>fahrenheit<|"|>,value:3}<|tool_response> ``` -**B. multiturn-thinking / unconstrained #2** — failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section +**B. multiturn-thinking / unconstrained #2** — FAIL ❌ (failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section) ``` <|tool_call>call:convert_temperature{from_unit:<|"|>celsius<|"|>,to_unit:<|"|>fahrenheit<|"|>,value:3}<|tool_response> ``` -**B. multiturn-thinking / unconstrained #3** — failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section +**B. multiturn-thinking / unconstrained #3** — FAIL ❌ (failed `stops_at_boundary`, `thought_ok`: ran past into <|tool_response> after the call; skipped or never closed the <|channel>thought section) ``` <|tool_call>call:convert_temperature{from_unit:<|"|>celsius<|"|>,to_unit:<|"|>fahrenheit<|"|>,value:3}<|tool_response> ``` -**C. adversarial-payload / unconstrained #0** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +### C. adversarial-payload + +**C. adversarial-payload / constrained #0** — PASS ✅ + +``` +<|tool_call>call:create_ticket{title:<|"|>Payments retry storm after config rollout<|"|>,body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} +Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42).<|"|>,severity:<|"|>blocker<|"|>,labels:[<|"|>bug<|"|>,<|"|>regression<|"|>,<|"|>backend<|"|>],affected:{service:<|"|>payments<|"|>,version:<|"|>2.14.1<|"|>,regions:[<|"|>ap-northeast-2<|"|>,<|"|>us-east-1<|"|>]},cc_emails:[<|"|>dev-alerts@example.com<|"|>],estimated_minutes:45}<|tool_call>call:escalate_incident{incident_id:<|"|>TICKET_ID_FROM_PREVIOUS_CALL<|"|>,level:1} +``` + +**C. adversarial-payload / constrained #1** — PASS ✅ + +``` +<|tool_call>call:create_ticket{ + title:<|"|>Payments retry storm after config rollout<|"|>, + body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} +Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42)<|"|>, + severity:<|"|>blocker<|"|>, + labels: [ + <|"|>bug<|"|>, + <|"|>regression<|"|>, + <|"|>backend<|"|> + ], + affected: { + service:<|"|>payments<|"|>, + version:<|"|>2.14.1<|"|>, + regions: [ + <|"|>ap-northeast-2<|"|>, + <|"|>us-east-1<|"|> + ] + }, + cc_emails: [ + <|"|>dev-alerts@example.com<|"|> + ], + estimated_minutes: 45 +}<|tool_call>call:escalate_incident{incident_id:<|"|>TICKET_ID<|"|>,level:1} +``` + +**C. adversarial-payload / constrained #2** — PASS ✅ + +``` +<|tool_call>call:create_ticket{ + title:<|"|>Payments retry storm after config rollout<|"|>, + body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} +Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42)<|"|>, + severity:<|"|>blocker<|"|>, + labels: [ + <|"|>bug<|"|>, + <|"|>regression<|"|>, + <|"|>backend<|"|> + ], + affected: { + service:<|"|>payments<|"|>, + version:<|"|>2.14.1<|"|>, + regions: [ + <|"|>ap-northeast-2<|"|>, + <|"|>us-east-1<|"|> + ] + }, + cc_emails: [ + <|"|>dev-alerts@example.com<|"|> + ], + estimated_minutes: 45 +}<|tool_call>call:escalate_incident{incident_id:<|"|>(id from create_ticket response)<|"|>,level:1} +``` + +**C. adversarial-payload / constrained #3** — PASS ✅ + +``` +<|tool_call>call:create_ticket{ + title:<|"|>Payments retry storm after config rollout<|"|>, + body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} +Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42)<|"|>, + severity:<|"|>blocker<|"|>, + labels: [ + <|"|>bug<|"|>, + <|"|>regression<|"|>, + <|"|>backend<|"|> + ], + affected: { + service:<|"|>payments<|"|>, + version:<|"|>2.14.1<|"|>, + regions: [ + <|"|>ap-northeast-2<|"|>, + <|"|>us-east-1<|"|> + ] + }, + cc_emails: [ + <|"|>dev-alerts@example.com<|"|> + ], + estimated_minutes: 45 +}<|tool_call>call:escalate_incident{incident_id:<|"|>[created_ticket_id]<|"|>,level:1} +``` + +**C. adversarial-payload / unconstrained #0** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_ticket{affected:{regions:[<|"|>ap-northeast-2<|"|>,<|"|>us-east-1<|"|>],service:<|"|>payments<|"|>,version:<|"|>2.14.1<|"|>},body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42).<|"|>,cc_emails:[<|"|>dev-alerts@example.com<|"|>],estimated_minutes:45,labels:[<|"|>regression<|"|>,<|"|>backend<|"|>],severity:<|"|>blocker<|"|>,title:<|"|>Payments retry storm after config rollout<|"|>,urgent_escalation:true}<|tool_response> ``` -**C. adversarial-payload / unconstrained #1** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**C. adversarial-payload / unconstrained #1** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_ticket{affected:{regions:[<|"|>ap-northeast-2<|"|>,<|"|>us-east-1<|"|>],service:<|"|>payments<|"|>,version:<|"|>2.14.1<|"|>},body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42).<|"|>,cc_emails:[<|"|>dev-alerts@example.com<|"|>],estimated_minutes:45,labels:[<|"|>bug<|"|>,<|"|>regression<|"|>,<|"|>backend<|"|>],severity:<|"|>blocker<|"|>,title:<|"|>Payments retry storm after config rollout<|"|>,urgent_escalation:true}<|tool_response> ``` -**C. adversarial-payload / unconstrained #2** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**C. adversarial-payload / unconstrained #2** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_ticket{affected:{regions:[<|"|>ap-northeast-2<|"|>,<|"|>us-east-1<|"|>],service:<|"|>payments<|"|>,version:<|"|>2.14.1<|"|>},body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} Error line: TypeError: cannot destructure {id: undefined} at applyRetry (retry.js:42)<|"|>,cc_emails:[<|"|>dev-alerts@example.com<|"|>],estimated_minutes:45,labels:[<|"|>bug<|"|>,<|"|>regression<|"|>,<|"|>backend<|"|>],severity:<|"|>blocker<|"|>,title:<|"|>Payments retry storm after config rollout<|"|>,urgent_escalation:true}<|tool_response> ``` -**C. adversarial-payload / unconstrained #3** — failed `stops_at_boundary`: ran past into <|tool_response> after the call +**C. adversarial-payload / unconstrained #3** — FAIL ❌ (failed `stops_at_boundary`: ran past into <|tool_response> after the call) ``` <|tool_call>call:create_ticket{affected:{regions:[<|"|>ap-northeast-2<|"|>,<|"|>us-east-1<|"|>],service:<|"|>payments<|"|>,version:<|"|>2.14.1<|"|>},body:<|"|>Deployed config: {"retry": {"max": 3, "backoff_ms": [100, 200], "jitter": true}} diff --git a/test_gemma4_scenarios.py b/test_gemma4_scenarios.py index 48ad496..52bdae9 100644 --- a/test_gemma4_scenarios.py +++ b/test_gemma4_scenarios.py @@ -391,7 +391,7 @@ def run_scenario(scenario, model, tokenizer, args): temperature = scenario.get("temperature", args.temperature) top_p = scenario.get("top_p", 0.95) summary = {} - failures = [] + samples = [] for mode in ["constrained", "unconstrained"]: counts = {k: 0 for k in metric_keys} print( @@ -418,23 +418,23 @@ def run_scenario(scenario, model, tokenizer, args): for k in metric_keys: counts[k] += bool(score[k]) failed_keys = [k for k in metric_keys if not score[k]] - if failed_keys: - reasons = [] - if score["error"]: - reasons.append(score["error"]) - if "stops_at_boundary" in failed_keys: - reasons.append("ran past into <|tool_response> after the call") - if "thought_ok" in failed_keys: - reasons.append("skipped or never closed the <|channel>thought section") - failures.append( - { - "mode": mode, - "sample": s, - "failed": failed_keys, - "reason": "; ".join(reasons), - "output": gen, - } - ) + reasons = [] + if score["error"]: + reasons.append(score["error"]) + if "stops_at_boundary" in failed_keys: + reasons.append("ran past into <|tool_response> after the call") + if "thought_ok" in failed_keys: + reasons.append("skipped or never closed the <|channel>thought section") + samples.append( + { + "mode": mode, + "sample": s, + "passed": not failed_keys, + "failed": failed_keys, + "reason": "; ".join(reasons), + "output": gen, + } + ) flags = " ".join(f"{k}={'O' if score[k] else 'X'}" for k in metric_keys) print(f"\n[{mode} #{s}] ({time.monotonic() - t0:.0f}s) {flags}") if score["error"]: @@ -449,7 +449,7 @@ def run_scenario(scenario, model, tokenizer, args): return { "summary": summary, "metric_keys": metric_keys, - "failures": failures, + "samples": samples, "temperature": temperature, "top_p": top_p, "max_new_tokens": max_new_tokens, @@ -496,15 +496,20 @@ def write_report(path, model_id, args, results): lines.append(f"- `{k}` — {desc}") lines.append("") - all_failures = [(name, f) for name, res in results.items() for f in res["failures"]] - if all_failures: - lines += ["## Failure examples", ""] - for name, f in all_failures[:12]: - failed = ", ".join(f"`{k}`" for k in f["failed"]) - lines.append(f"**{name} / {f['mode']} #{f['sample']}** — failed {failed}: {f['reason']}") + lines += ["## Sample outputs (raw)", ""] + for name, res in results.items(): + lines.append(f"### {name}") + lines.append("") + for s in res["samples"]: + status = "PASS ✅" if s["passed"] else "FAIL ❌" + header = f"**{name} / {s['mode']} #{s['sample']}** — {status}" + if not s["passed"]: + failed = ", ".join(f"`{k}`" for k in s["failed"]) + header += f" (failed {failed}: {s['reason']})" + lines.append(header) lines.append("") lines.append("```") - lines.append(f["output"][:900]) + lines.append(s["output"][:900]) lines.append("```") lines.append("")