.

test_gemma4_scenarios.py

@@ -391,7 +391,7 @@ def run_scenario(scenario, model, tokenizer, args):
     temperature = scenario.get("temperature", args.temperature)
     top_p = scenario.get("top_p", 0.95)
     summary = {}
-    failures = []
+    samples = []
     for mode in ["constrained", "unconstrained"]:
         counts = {k: 0 for k in metric_keys}
         print(
@@ -418,23 +418,23 @@ def run_scenario(scenario, model, tokenizer, args):
             for k in metric_keys:
                 counts[k] += bool(score[k])
             failed_keys = [k for k in metric_keys if not score[k]]
-            if failed_keys:
-                reasons = []
-                if score["error"]:
-                    reasons.append(score["error"])
-                if "stops_at_boundary" in failed_keys:
-                    reasons.append("ran past <tool_call|> into <|tool_response> after the call")
-                if "thought_ok" in failed_keys:
-                    reasons.append("skipped or never closed the <|channel>thought section")
-                failures.append(
-                    {
-                        "mode": mode,
-                        "sample": s,
-                        "failed": failed_keys,
-                        "reason": "; ".join(reasons),
-                        "output": gen,
-                    }
-                )
+            reasons = []
+            if score["error"]:
+                reasons.append(score["error"])
+            if "stops_at_boundary" in failed_keys:
+                reasons.append("ran past <tool_call|> into <|tool_response> after the call")
+            if "thought_ok" in failed_keys:
+                reasons.append("skipped or never closed the <|channel>thought section")
+            samples.append(
+                {
+                    "mode": mode,
+                    "sample": s,
+                    "passed": not failed_keys,
+                    "failed": failed_keys,
+                    "reason": "; ".join(reasons),
+                    "output": gen,
+                }
+            )
             flags = " ".join(f"{k}={'O' if score[k] else 'X'}" for k in metric_keys)
             print(f"\n[{mode} #{s}] ({time.monotonic() - t0:.0f}s) {flags}")
             if score["error"]:
@@ -449,7 +449,7 @@ def run_scenario(scenario, model, tokenizer, args):
     return {
         "summary": summary,
         "metric_keys": metric_keys,
-        "failures": failures,
+        "samples": samples,
         "temperature": temperature,
         "top_p": top_p,
         "max_new_tokens": max_new_tokens,
@@ -496,15 +496,20 @@ def write_report(path, model_id, args, results):
         lines.append(f"- `{k}` — {desc}")
     lines.append("")
 
-    all_failures = [(name, f) for name, res in results.items() for f in res["failures"]]
-    if all_failures:
-        lines += ["## Failure examples", ""]
-        for name, f in all_failures[:12]:
-            failed = ", ".join(f"`{k}`" for k in f["failed"])
-            lines.append(f"**{name} / {f['mode']} #{f['sample']}** — failed {failed}: {f['reason']}")
+    lines += ["## Sample outputs (raw)", ""]
+    for name, res in results.items():
+        lines.append(f"### {name}")
+        lines.append("")
+        for s in res["samples"]:
+            status = "PASS ✅" if s["passed"] else "FAIL ❌"
+            header = f"**{name} / {s['mode']} #{s['sample']}** — {status}"
+            if not s["passed"]:
+                failed = ", ".join(f"`{k}`" for k in s["failed"])
+                header += f" (failed {failed}: {s['reason']})"
+            lines.append(header)
             lines.append("")
             lines.append("```")
-            lines.append(f["output"][:900])
+            lines.append(s["output"][:900])
             lines.append("```")
             lines.append("")