From 95b258e3495c4f30a1185b471780e1a30b564da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Sch=C3=BCller?= Date: Mon, 10 Aug 2026 12:16:45 +0200 Subject: [PATCH 01/50] feat(experimentalist): add a fast, diagnostic smoke agent fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing examples take upwards of two hours per run and produce stochastic outcomes, so they can confirm the loop completed but never that it improved anything. This is a fixture where the expected result is known in advance, which makes a run checkable rather than merely plausible. The agent makes no model calls -- handlers are regular expressions over a small records file -- so the only stochastic component in a run is the Experimentalist itself, and its CompletionClient points at an unroutable address so an accidental model call fails loudly rather than quietly making the agent nondeterministic. One prebuilt image serves every task, tagged by a content hash of the Dockerfile and records, so a forgotten rebuild fails a test instead of silently running against stale data. Five task groups, each answering a different question about the loop: a repair that a general fix reaches, a generalization trap where the tempting fix passes train and fails validation, a pattern too narrow, a clipped instruction, and a fix that needs several coordinated edits. Every group carries a control that passes at baseline, so a destructive fix costs reward instead of going unnoticed. Which split a run uses lives on the profile, not the config: EvolutionaryOptimizerConfig has no dataset field, so a scenario config cannot name its own tasks. Both single-round scenarios therefore share configs/single-round.yaml, and optimizer-generalization.yaml selects the generalization split. The profile is the right home for this, because which split a run uses is what decides whether the winner should beat the baseline or the baseline should be retained -- that is profile-level information rather than tuning. Also records why the task fixtures carry no licence header, so the question does not get reopened: instruction.md is the prompt and expected.txt is compared byte-for-byte, so a header would change what the agent reads and what it is scored against. Signed-off-by: Christian Schüller Signed-off-by: Gaia Di Lorenzo --- .../docs/smoke-agent-weaknesses.md | 151 +++++++++ .../examples/smoke-agent/.gitignore | 7 + .../examples/smoke-agent/AGENT-SPEC.md | 52 +++ .../examples/smoke-agent/README.md | 248 ++++++++++++++ .../examples/smoke-agent/agent/agent.py | 103 ++++++ .../smoke-agent/agent/harbor_wrapper.py | 125 +++++++ .../examples/smoke-agent/agent/main.py | 33 ++ .../examples/smoke-agent/configs/full.yaml | 76 +++++ .../smoke-agent/configs/single-round.yaml | 39 +++ .../smoke-agent/dataset/_shared/Dockerfile | 25 ++ .../smoke-agent/dataset/_shared/records.json | 8 + .../smoke-agent/dataset/_shared/test.sh | 55 ++++ .../train/lookup-ada/environment/.gitkeep | 0 .../train/lookup-ada/instruction.md | 9 + .../g1-aggregation/train/lookup-ada/task.toml | 27 ++ .../train/lookup-ada/tests/expected.txt | 1 + .../train/lookup-ada/tests/test.sh | 55 ++++ .../environment/.gitkeep | 0 .../total-hours-engineers/instruction.md | 9 + .../train/total-hours-engineers/task.toml | 27 ++ .../total-hours-engineers/tests/expected.txt | 1 + .../train/total-hours-engineers/tests/test.sh | 55 ++++ .../total-hours-research/environment/.gitkeep | 0 .../train/total-hours-research/instruction.md | 9 + .../train/total-hours-research/task.toml | 27 ++ .../total-hours-research/tests/expected.txt | 1 + .../train/total-hours-research/tests/test.sh | 55 ++++ .../lookup-grace/environment/.gitkeep | 0 .../validation/lookup-grace/instruction.md | 9 + .../validation/lookup-grace/task.toml | 27 ++ .../lookup-grace/tests/expected.txt | 1 + .../validation/lookup-grace/tests/test.sh | 55 ++++ .../total-hours-analysts/environment/.gitkeep | 0 .../total-hours-analysts/instruction.md | 9 + .../validation/total-hours-analysts/task.toml | 27 ++ .../total-hours-analysts/tests/expected.txt | 1 + .../total-hours-analysts/tests/test.sh | 55 ++++ .../total-hours-ops/environment/.gitkeep | 0 .../validation/total-hours-ops/instruction.md | 9 + .../validation/total-hours-ops/task.toml | 27 ++ .../total-hours-ops/tests/expected.txt | 1 + .../validation/total-hours-ops/tests/test.sh | 55 ++++ .../train/lookup-ada/environment/.gitkeep | 0 .../train/lookup-ada/instruction.md | 9 + .../train/lookup-ada/task.toml | 28 ++ .../train/lookup-ada/tests/expected.txt | 1 + .../train/lookup-ada/tests/test.sh | 55 ++++ .../train/lookup-obrien/environment/.gitkeep | 0 .../train/lookup-obrien/instruction.md | 9 + .../train/lookup-obrien/task.toml | 28 ++ .../train/lookup-obrien/tests/expected.txt | 1 + .../train/lookup-obrien/tests/test.sh | 55 ++++ .../train/lookup-zoe/environment/.gitkeep | 0 .../train/lookup-zoe/instruction.md | 9 + .../train/lookup-zoe/task.toml | 28 ++ .../train/lookup-zoe/tests/expected.txt | 1 + .../train/lookup-zoe/tests/test.sh | 55 ++++ .../lookup-ann-marie/environment/.gitkeep | 0 .../lookup-ann-marie/instruction.md | 9 + .../validation/lookup-ann-marie/task.toml | 28 ++ .../lookup-ann-marie/tests/expected.txt | 1 + .../validation/lookup-ann-marie/tests/test.sh | 55 ++++ .../lookup-grace/environment/.gitkeep | 0 .../validation/lookup-grace/instruction.md | 9 + .../validation/lookup-grace/task.toml | 28 ++ .../lookup-grace/tests/expected.txt | 1 + .../validation/lookup-grace/tests/test.sh | 55 ++++ .../lookup-role-obrien/environment/.gitkeep | 0 .../lookup-role-obrien/instruction.md | 9 + .../validation/lookup-role-obrien/task.toml | 28 ++ .../lookup-role-obrien/tests/expected.txt | 1 + .../lookup-role-obrien/tests/test.sh | 55 ++++ .../train/plain-dept/environment/.gitkeep | 0 .../train/plain-dept/instruction.md | 9 + .../g3-long-inputs/train/plain-dept/task.toml | 28 ++ .../train/plain-dept/tests/expected.txt | 1 + .../train/plain-dept/tests/test.sh | 55 ++++ .../train/preamble-dept/environment/.gitkeep | 0 .../train/preamble-dept/instruction.md | 9 + .../train/preamble-dept/task.toml | 28 ++ .../train/preamble-dept/tests/expected.txt | 1 + .../train/preamble-dept/tests/test.sh | 55 ++++ .../train/preamble-role/environment/.gitkeep | 0 .../train/preamble-role/instruction.md | 9 + .../train/preamble-role/task.toml | 28 ++ .../train/preamble-role/tests/expected.txt | 1 + .../train/preamble-role/tests/test.sh | 55 ++++ .../preamble-hours/environment/.gitkeep | 0 .../validation/preamble-hours/instruction.md | 9 + .../validation/preamble-hours/task.toml | 28 ++ .../preamble-hours/tests/expected.txt | 1 + .../validation/preamble-hours/tests/test.sh | 55 ++++ .../preamble-long-dept/environment/.gitkeep | 0 .../preamble-long-dept/instruction.md | 9 + .../validation/preamble-long-dept/task.toml | 28 ++ .../preamble-long-dept/tests/expected.txt | 1 + .../preamble-long-dept/tests/test.sh | 55 ++++ .../trailing-prose/environment/.gitkeep | 0 .../validation/trailing-prose/instruction.md | 9 + .../validation/trailing-prose/task.toml | 28 ++ .../trailing-prose/tests/expected.txt | 1 + .../validation/trailing-prose/tests/test.sh | 55 ++++ .../train/count-ops/environment/.gitkeep | 0 .../train/count-ops/instruction.md | 9 + .../train/count-ops/task.toml | 28 ++ .../train/count-ops/tests/expected.txt | 1 + .../train/count-ops/tests/test.sh | 55 ++++ .../train/count-research/environment/.gitkeep | 0 .../train/count-research/instruction.md | 9 + .../train/count-research/task.toml | 28 ++ .../train/count-research/tests/expected.txt | 1 + .../train/count-research/tests/test.sh | 55 ++++ .../train/lookup-ada/environment/.gitkeep | 0 .../train/lookup-ada/instruction.md | 9 + .../train/lookup-ada/task.toml | 28 ++ .../train/lookup-ada/tests/expected.txt | 1 + .../train/lookup-ada/tests/test.sh | 55 ++++ .../environment/.gitkeep | 0 .../count-engineers-research/instruction.md | 9 + .../count-engineers-research/task.toml | 28 ++ .../tests/expected.txt | 1 + .../count-engineers-research/tests/test.sh | 55 ++++ .../count-operators-ops/environment/.gitkeep | 0 .../count-operators-ops/instruction.md | 9 + .../validation/count-operators-ops/task.toml | 28 ++ .../count-operators-ops/tests/expected.txt | 1 + .../count-operators-ops/tests/test.sh | 55 ++++ .../lookup-grace/environment/.gitkeep | 0 .../validation/lookup-grace/instruction.md | 9 + .../validation/lookup-grace/task.toml | 28 ++ .../lookup-grace/tests/expected.txt | 1 + .../validation/lookup-grace/tests/test.sh | 55 ++++ .../train/empty-role/environment/.gitkeep | 0 .../train/empty-role/instruction.md | 9 + .../g5-edge-cases/train/empty-role/task.toml | 28 ++ .../train/empty-role/tests/expected.txt | 1 + .../train/empty-role/tests/test.sh | 55 ++++ .../train/lookup-ada/environment/.gitkeep | 0 .../train/lookup-ada/instruction.md | 9 + .../g5-edge-cases/train/lookup-ada/task.toml | 28 ++ .../train/lookup-ada/tests/expected.txt | 1 + .../train/lookup-ada/tests/test.sh | 55 ++++ .../train/missing-person/environment/.gitkeep | 0 .../train/missing-person/instruction.md | 9 + .../train/missing-person/task.toml | 28 ++ .../train/missing-person/tests/expected.txt | 1 + .../train/missing-person/tests/test.sh | 55 ++++ .../lookup-grace/environment/.gitkeep | 0 .../validation/lookup-grace/instruction.md | 9 + .../validation/lookup-grace/task.toml | 28 ++ .../lookup-grace/tests/expected.txt | 1 + .../validation/lookup-grace/tests/test.sh | 55 ++++ .../missing-person-hours/environment/.gitkeep | 0 .../missing-person-hours/instruction.md | 9 + .../validation/missing-person-hours/task.toml | 28 ++ .../missing-person-hours/tests/expected.txt | 1 + .../missing-person-hours/tests/test.sh | 55 ++++ .../missing-person-role/environment/.gitkeep | 0 .../missing-person-role/instruction.md | 9 + .../validation/missing-person-role/task.toml | 28 ++ .../missing-person-role/tests/expected.txt | 1 + .../missing-person-role/tests/test.sh | 55 ++++ .../dataset/insights/g1-aggregation.yaml | 47 +++ .../dataset/task-template/README.md | 29 ++ .../task-template/environment/.gitkeep | 0 .../dataset/task-template/instruction.md | 9 + .../dataset/task-template/task.toml | 27 ++ .../dataset/task-template/tests/expected.txt | 1 + .../dataset/task-template/tests/test.sh | 55 ++++ .../smoke-agent/optimizer-generalization.yaml | 37 +++ .../examples/smoke-agent/optimizer.yaml | 33 ++ .../smoke-agent/scripts/build_all_group.py | 98 ++++++ .../smoke-agent/scripts/build_image.py | 93 ++++++ .../smoke-agent/scripts/record_traces.py | 157 +++++++++ .../smoke-agent/scripts/sync_verifier.py | 49 +++ .../tests/experimentalist/test_smoke_agent.py | 168 ++++++++++ .../test_smoke_agent_assets.py | 251 ++++++++++++++ .../test_smoke_agent_baseline.py | 208 ++++++++++++ .../experimentalist/test_smoke_agent_gate.py | 311 ++++++++++++++++++ .../test_smoke_agent_traces.py | 79 +++++ 180 files changed, 5358 insertions(+) create mode 100644 plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/.gitignore create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/README.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/configs/single-round.yaml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/environment/.gitkeep create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt create mode 100755 plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py create mode 100644 plugins/nemo-experimentalist/examples/smoke-agent/scripts/sync_verifier.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_traces.py diff --git a/plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md b/plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md new file mode 100644 index 0000000000..c268c43825 --- /dev/null +++ b/plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md @@ -0,0 +1,151 @@ + + + +# smoke-agent: the deliberate weaknesses + +`examples/smoke-agent` ships an agent that is **wrong on purpose**. Five known +weaknesses, each paired with a group of Harbor tasks that surfaces it, so an +Experimentalist run can be asserted to have *repaired* something rather than +merely completed. + +**Do not fix them in the agent.** A well-meaning cleanup silently destroys what +the fixture measures: with the weakness gone, the baseline passes, the Analyzer +gets no failing trace, and a run that does nothing looks identical to a run that +works. + +## Why this file is here and not next to the agent + +`get_agent_code` copies the example directory into `source-agent/` minus +`_AGENT_COPY_EXCLUDE_NAMES` (`__pycache__`, `.git`, `.claude`, `.uv`, `.venv`, +`artifacts`, `dataset`, `eval-and-optimize`, `scratch`). **Everything else under +the example root is readable by the Coder**, including `AGENT-SPEC.md`, which +`coder.py` reads before any source file. + +So a description of the weaknesses inside the example would hand the Coder the +diagnosis the fixture exists to test, and a run could pass with a broken +Analyzer. That is why `agent.py` carries only ordinary engineering comments, +`AGENT-SPEC.md` has no known-gaps section, and the frozen Insights live under +`dataset/` — the one in-example directory the copy excludes. + +There is evidence this matters and works: the analyst diagnosed G1 correctly from +traces alone, with no hint available in the source or the spec. + +## The five weaknesses + +All line references are to `examples/smoke-agent/agent/agent.py`. + +### G1 — no aggregation capability + +**What.** Nothing sums or averages a numeric field. `solve` dispatches over +`handle_lookup`, `handle_list`, `handle_count`; none matches "what is the total +`` for …", so every aggregation question falls through to `FALLBACK`. + +**Odd one out.** This is the only *missing* capability; G2–G5 are flawed code +paths that exist. There is no wrong code to find, only absent code, so G1's tasks +exercise the Proposer more than the Coder. + +**Tasks.** `dataset/groups/g1-aggregation/` — train sums by department +(`total=29`, `total=13`); validation sums with no scope (`total=42`) and by +*role* (`total=20`). A fix that hardcodes a department filter passes train and +fails validation. That is the generalization axis. + +**A correct repair** adds a handler that sums a field over a record subset, with +the scope taken from the question rather than assumed, and inserts it into the +dispatch tuple. + +### G2 — name pattern too narrow + +**What.** `LOOKUP_RE`'s name group is `([A-Za-z ]+)`, so any name carrying an +apostrophe, a hyphen, or a non-ASCII character fails to match at all and falls +through to `FALLBACK`. + +**Tasks.** `dataset/groups/g2-name-patterns/` — `O'Brien`, `Zoë Washington`, +`Ann-Marie Cruz`. Controls are plain-ASCII lookups that already work. + +**A correct repair** widens the character class. Note `str.isalpha()` is *not* a +valid test for "would this name match" — it is Unicode-aware and accepts `Zoë` +while the agent's ASCII class rejects it. The guard test uses the agent's own +class for exactly this reason. + +### G3 — instruction clipped before dispatch + +**What.** `solve` truncates to `MAX_INSTRUCTION_CHARS = 240` before dispatching, +so a question preceded by a long preamble is cut off and matches nothing. + +**Tasks.** `dataset/groups/g3-long-inputs/` — a ~300-character reporting-policy +preamble in front of a question that works without it. + +**The control is load-bearing.** `trailing-prose` puts the question *first* and +the prose after, so it passes at baseline **and** would break under a fix that +reads only the tail of a clipped instruction. It catches a specific bad repair. + +**A correct repair** raises or removes the limit. + +### G4 — dispatch order shadows the count handler + +**What.** `LIST_RE` is `(?:list|how many) .*? in the (\w+) department` — the +`how many` alternative belongs to `COUNT_RE` — and `solve` consults +`handle_list` first. Counting questions are therefore answered with a list of +names. + +**Odd one out.** This is the only group whose failure is a *wrong-shaped answer* +rather than the fallback, so it scores `reward 0` with `shape_ok 1.0`. G1, G2, +G3 and G5's missing-record mode all fall back to prose and score `shape_ok 0`. +Measured: G4 train aggregates to `reward 0.333, shape_ok 1.0`, G5 validation to +`0.333, 0.333`. The two groups are separable from aggregates alone. + +**Two halves, one repair.** `LIST_RE` and the dispatch tuple order are a matched +pair. Changing either alone leaves the shadowing in place. + +**Tasks.** `dataset/groups/g4-dispatch-order/` — train counts people per +department; validation counts *by role within* a department, which matches +`LIST_RE` but **not** `COUNT_RE`. So reordering the tuple alone still fails +validation; the count pattern has to widen too. + +### G5 — missing and empty data not handled + +**What.** Two distinct failure modes: + +- `handle_lookup` resolves a record with `next(r for r in … if r["name"] == name)`, + which **raises** when the name is absent. `solve`'s top-level `except` catches + it and returns `FALLBACK`. +- An empty stored value yields an empty right-hand side (`role=`) rather than a + documented `role=unknown`. + +**Why the catch exists.** Without it the process exits non-zero, the wrapper +raises, and Harbor records a *harness error* rather than a scored 0 — which +leaves the Analyzer nothing to read. Verified: G5 trials complete with +`status=completed` and reward 0. The catch controls the exit code, not the trace; +NOOA already records the exception on the failing handler's span. + +**Tasks.** `dataset/groups/g5-edge-cases/` — a name absent from the records, and +Karl Jung's empty `role`. Expected answers use `unknown`. + +## Orthogonality, including the data + +Each group's tasks must supply evidence for **its own weakness only**, so a run's +failing traces point at one root cause. That extends to `records.json`, which all +five groups share — the data is a coupling surface as much as the code is. + +Current assignment, pinned by `test_smoke_agent_baseline.py`: + +| Record | Serves | Must stay | +| --- | --- | --- | +| Ada Lovelace, Grace Hopper | controls, G1 engineer sum | plain ASCII, non-empty, int hours | +| Zoë Washington, O'Brien, Ann-Marie Cruz | G2 | non-empty `role`, int `hours` | +| Karl Jung | G5 empty-field mode | empty `role`, **int `hours`** | + +Karl Jung's `hours` is `0`, not empty, and that is deliberate: an empty `hours` +sits in `ops`, which G1 aggregates, so it would force a G1 fix to absorb G5's +robustness. + +This has already gone wrong once through a *task* rather than the data. G3's +`preamble-long-dept` originally looked up a name absent from the records, so +closing G3 alone would have left it failing on G5's missing-record path. Check +both when adding either. + +## If a guard test fails + +`test_smoke_agent_baseline.py` and `test_smoke_agent.py` pin every behaviour +above. A failure almost always means the agent was "fixed" rather than that the +tests are wrong. Confirm against this document before changing either. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore b/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore new file mode 100644 index 0000000000..75f9fc9d3c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore @@ -0,0 +1,7 @@ +# Generated by scripts/build_all_group.py, not authored. +# +# The loop takes a single --train-dataset path, so running several groups at once +# needs one combined directory. Its contents are byte-identical copies of the +# other groups, so committing it would double the dataset in the repo and put +# every group change in two places. Build it before running the full scenario. +dataset/groups/_all/ diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md b/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md new file mode 100644 index 0000000000..ad86acd57f --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md @@ -0,0 +1,52 @@ + + + +# smoke-agent + +## Job + +Answer one question about the records file at `/app/data/records.json` and write +the single answer line to `/app/artifacts/output.txt`. + +## Interface + +- Invoked as `python main.py --prompt ""` with `/app` as the + working directory. +- Writes exactly one line, plus a trailing newline, to + `/app/artifacts/output.txt`. +- Writes an OTLP JSONL trace under `/app/traces/`. + +## Design + +`ReportAgent.solve` dispatches the instruction across an ordered list of +handlers and returns the first non-`None` answer, falling back to a fixed +string. Each handler matches the question with a regular expression, looks the +answer up in the records, and formats one line as `=`. + +The records are a list of objects with `name`, `dept`, `role`, and `hours`. +`FIELD_ALIASES` maps the word a question uses to the key the records store it +under, so the answer line is always keyed by the canonical field name. + +## Missing and empty values + +A question may name a person the records do not contain, or ask for a field +whose stored value is an empty string. Both are answered the same way: the value +is the word `unknown`, so the line reads `dept=unknown`. This is part of the +output contract and is compared byte-for-byte like any other answer — the +sentinel is `unknown` exactly, not `n/a`, `none`, or the empty string. + +## Constraints — these are hard requirements + +- **The agent is deterministic and offline.** The same instruction must always + produce the same answer. Reward differences between candidates must come from + code changes, never from sampling. +- **No LLM.** Do not add a `@strategy` method, an LLM-backed handler, a subagent + with its own model, or a model swap. The task container has no network and no + API key, so such a change fails outright — but more importantly, being + reproducible is this agent's entire contract. +- Standard library plus NOOA only. No new dependencies. +- Do not edit `/app/data/records.json`. It is task-supplied input, not agent + code, and it is not part of this directory. +- The output line is compared byte-for-byte against the task's expected value, + so trailing whitespace, extra lines, and changes to the `=` form + all count as wrong answers. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/README.md b/plugins/nemo-experimentalist/examples/smoke-agent/README.md new file mode 100644 index 0000000000..c9f5f69f1d --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/README.md @@ -0,0 +1,248 @@ + + + +# smoke-agent + +A fast fixture for exercising the Experimentalist loop end to end. It exists to +make a full round cheap enough to run while refactoring, and to let a run be +checked for more than "it completed". + +> **Nothing inside `agent/` may describe what this fixture measures.** That +> directory is what `agent_source` points at, so it is copied into every +> candidate workspace and read by the Coder; a description there would hand it +> the diagnosis the fixture exists to test. Everything else here — this file, +> `configs/`, `scripts/`, `dataset/` — is never copied and can say whatever is +> useful. `AGENT-SPEC.md` reaches the LLM components by a separate route and is +> held to the same rule as `agent/`. +> +> Why a baseline must not change, and the per-group detail, live in +> `plugins/nemo-experimentalist/docs/`. + +## Design + +- **The agent makes no model calls.** Handlers are regular expressions plus a + dict lookup over `dataset/_shared/records.json`, so the same instruction always + produces the same answer and the only stochastic component in a run is the + Experimentalist itself. The `CompletionClient` in `agent.py` points at an + unroutable address: an accidental model call fails loudly rather than quietly + making the agent nondeterministic. +- **Tasks are local and checked in.** No registry, no NeMo Platform for Mode 2, + no network inside the task container. +- **One prebuilt image serves every task**, referenced by + `[environment].docker_image` rather than a per-task Dockerfile. Its tag is a + content hash of the Dockerfile and the records file, so forgetting to rebuild + fails a test instead of silently running against stale data. + +## Layout + +```text +agent/ ONLY this is copied to the Coder (agent_source) +agent/agent.py the code under optimization +agent/main.py container entry point +agent/harbor_wrapper.py Harbor upload + exec adapter +AGENT-SPEC.md behaviour contract read by the LLM components +optimizer.yaml profile: agent source, spec, g1 datasets +optimizer-generalization.yaml profile: same agent, g4 datasets (see Scenarios) +configs/single-round.yaml loop settings for both single-round scenarios +configs/full.yaml loop settings for the multi-round scenario +dataset/_shared/ canonical Dockerfile, records, verifier +dataset/groups// train/ and validation/ task sets +dataset/groups/_all/ GENERATED, gitignored -- see build_all_group.py +dataset/task-template/ insight mode only +dataset/insights/ insight mode only, frozen analyst output +scripts/build_image.py build the image, stamp its tag into every task +scripts/sync_verifier.py copy the canonical verifier into every task +scripts/build_all_group.py assemble the combined group the full scenario runs +scripts/record_traces.py evaluate a group and ingest its traces +``` + +## Running it + +Run the loop **inside a Docker sandbox**. Clone mode gives it a private writable +clone rather than write access to the host checkout, which keeps a run from +touching this directory. + +```bash +repo="$(git rev-parse --show-toplevel)" + +sbx create --clone --name nemo-experimentalist shell "$repo" +``` + +> **In a git worktree?** `--clone` refuses to run there, so bind-mount instead: +> +> ```bash +> sbx create --profile developer --name nemo-experimentalist shell "$(pwd)" +> ``` +> +> That gives the sandbox write access to this checkout, so the source-mutation +> protection clone mode provides is gone. Check `git status` on the example after +> a run. + +```bash +# The image lives in the sandbox's own Docker daemon, so build it there. +sbx exec --workdir "$repo" nemo-experimentalist bash -lc \ + 'cd plugins/nemo-experimentalist/examples/smoke-agent && uv run --no-project scripts/build_image.py' +``` + +Then run a scenario. `--with ./plugins/nemo-agents` is required: the `agents` +command group lives in a separate workspace package, and without it the CLI fails +with `No module named 'nemo_agents_plugin'`. + +```bash +sbx exec --workdir "$repo" \ + --env UV_PROJECT_ENVIRONMENT=/home/agent/.venvs/nemo-platform \ + --env INFERENCE_API_KEY \ + --env NEMO_EXPERIMENTALIST_MODELS_SMART=openai/openai/openai/gpt-5.5 \ + --env NEMO_EXPERIMENTALIST_MODELS_MID=openai/openai/openai/gpt-5-mini \ + --env NEMO_EXPERIMENTALIST_MODELS_FAST=openai/openai/openai/gpt-5-mini \ + nemo-experimentalist \ + bash -lc 'uv run --frozen --python 3.13 \ + --package nemo-experimentalist-plugin --with ./plugins/nemo-agents \ + nemo agents experimentalist run \ + --profile plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml \ + --no-insight \ + --config plugins/nemo-experimentalist/examples/smoke-agent/configs/single-round.yaml \ + --experiment-dir /tmp/smoke-repair' +``` + +The model names above are examples — substitute whatever your endpoint calls its +models. The tiers have no defaults: `model_name()` raises rather than guessing, +since a model name is only meaningful against a specific endpoint. The leading +`openai/` is litellm routing rather than part of the name, and dropping it makes +a valid endpoint id fail inside the client. + +The tier variables are `NEMO_EXPERIMENTALIST_MODELS_{SMART,MID,FAST}` — nested +config fields join with `_`, so a `_NAME` suffix is silently ignored rather than +rejected, and the run then fails at the first LLM call. + +Copy the experiment directory back out with `sbx cp` to check the result. + +## Scenarios + +**The profile picks the scenario, not the config.** What separates a repair run +from a generalization one is the split it runs against, so each has its own +profile: + +| Profile | Config | Rounds | A healthy run ends with | +| --- | --- | --- | --- | +| `optimizer.yaml` | `single-round.yaml` | 1 | the winner beating the baseline | +| `optimizer-generalization.yaml` | `single-round.yaml` | 1 | the baseline correctly retained | +| `optimizer.yaml` | `full.yaml` | up to 5 | every task in the combined group passing | + +The first two are opposite tests, so a run is only meaningful once you know which +one you started — and the config cannot tell you, because both use the same one. +That is not an oversight: the scenario config carries loop settings only, and the +schema has no dataset field at all. Which split a run uses lives on the profile, +so the profile is what decides the question. + +```bash +# generalization: same agent, held-out split, baseline expected to win +--profile optimizer-generalization.yaml --config configs/single-round.yaml +``` + +`full.yaml` is the only one that exercises the evolutionary machinery — survivors +carried between rounds, ranking over more than two candidates, and the +convergence check. It runs against `dataset/groups/_all`, which is **generated +and gitignored**; build it first, or the run loads zero tasks and reports +`No tasks matched the filter(s)` rather than erroring: + +```bash +sbx exec --workdir "$repo" nemo-experimentalist bash -lc \ + 'cd plugins/nemo-experimentalist/examples/smoke-agent && uv run --no-project scripts/build_all_group.py' +``` + +Rerun that after changing any group. Not every group is in the combined set — +`build_all_group.py` says which are held out and why. + +## Why the task files carry no licence header + +`instruction.md` and `tests/expected.txt` under `dataset/groups/` are literal +payloads, not source. `expected.txt` is compared byte-for-byte, so a header would +become part of the expected answer and every task would fail; `instruction.md` is +the prompt handed to the agent, so a header would become part of the question. + +This matches the repository as it stands rather than carving out a new exception: +the `copyright-fix` hook is scoped to `\.(py|ts)$`, and comparable fixtures +elsewhere — `sdk/python/nemo-platform/tests/sample_file.txt`, the model-spec +`test_data/**/README.md` files, the `automodel` upload fixtures — carry no header +either. Every file here that *is* source does carry one. + +## Groups + +Every group is a self-contained train/validation pair of six tasks — per split, +two that fail at baseline and one that already passes. That control is the point: +it makes a destructive fix cost reward instead of passing unnoticed, and it means +a group's score can fall as well as rise. + +| Group | What a run against it tests | Backs | In `_all` | +| --- | --- | --- | --- | +| `g1-aggregation` | **Repair.** Train shows two kinds of filter, so a general fix is reachable — and a hardcoded one fails validation. | `single-round.yaml` | yes | +| `g2-name-patterns` | Widening a pattern that is too narrow. Train shows two kinds of awkward name, validation a third. | `full.yaml` | yes | +| `g3-long-inputs` | A constant rather than logic — the one `edit_config` in the set, so a run exercises a different path through the Coder. | `full.yaml` | yes | +| `g4-dispatch-order` | **Generalization.** The tempting fix passes train and fails validation, so a healthy run *keeps the baseline*. | `single-round.yaml` | no | +| `g5-edge-cases` | Several changes that score only when all are made, whose partial states are indistinguishable in the output. The hardest here. | `single-round.yaml` | no | + +Two groups are held out of the combined set, for different reasons. + +`g4-dispatch-order`'s healthy outcome — baseline retained — is the opposite of +the combined scenario's, and one run cannot assert both. + +`g5-edge-cases` is only reachable with trajectory scoring on, and the combined +scenario runs with it off: measured over runs made after the spec stated the +sentinel, no candidate closed it without a goal tree and most did with one. +Trajectory scoring is not dependable enough to leave on yet, so the group is out +until it is. `build_all_group.py` records the numbers and the condition for +putting it back. Run it on its own with `single-round.yaml` in the meantime — it is a +repair-shaped split. + +The groups are not interchangeable and not redundant. Each was built so that a +run against it answers a different question about the loop, which is why the +combined scenario wants several at once rather than more tasks from one: + +- Several independently addressable groups let a single round produce genuinely + *different* candidates, and let a later round inherit one fix and add another. + A one-group dataset can show neither. +- They differ in the kind of edit they call for, so a run exercises more than one + path through the Coder. +- The groups differ in difficulty. The easier ones give an early round something + to find, so a run that stalls later still shows the machinery working; the + harder ones keep the ceiling out of reach of a shallow fix. + +What any individual group measures, and why its baseline must not change, is +documented outside this directory — see the note at the top of this file. + +## Checking a run + +```bash +SMOKE_EXPERIMENT_DIR=/tmp/smoke-repair uv run pytest \ + plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py -v +``` + +Five assertions, each covering a different way a result can look better than it +is: the winner's code actually changed, reward improved beyond noise, controls +still pass, the analysis named the cause, and the winner handles held-out +instances rather than the ones it trained on. + +**Also run the guard suite after every loop run:** + +```bash +uv run pytest plugins/nemo-experimentalist/tests/experimentalist/ -k smoke -q +``` + +It pins this fixture's baseline behaviour. A failure there means the fixture +itself changed, which makes every later run meaningless while still looking +healthy. Do not "fix" the agent to make it pass; read the documents referenced at +the top first. + +## Timings + +Measured on 2026-08-05, one round, two candidates, three tasks per split: + +| | | +| --- | --- | +| One split evaluated in containers | ~15 s | +| A full round end to end | ~18 min | + +Container evaluation is negligible by design. The cost is the Experimentalist's +own components — the architecture doc, Analyzer, Proposer, and one Coder pass per +candidate — which is the part under test and cannot be optimized away here. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py new file mode 100644 index 0000000000..e1d300a1c8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Answer one question about a records file. + +Deterministic by construction: no LLM call, no network, standard library plus +NOOA only. The same instruction always produces the same answer. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path + +from nooa import Agent +from nooa.tracing import enable_tracing, exporters +from nooa.unifiedllm import CompletionClient + +enable_tracing(exporters=[exporters.jsonl(trace_dir=os.environ.get("TRACE_DIR", "/app/traces/"))]) + +logger = logging.getLogger(__name__) + +RECORDS_PATH = Path(os.environ.get("RECORDS_PATH", "/app/data/records.json")) +FALLBACK = "I do not know how to answer that." + +# Bound how much of the instruction we scan, so an oversized input cannot make +# the regex pass expensive. +MAX_INSTRUCTION_CHARS = 240 + +# Maps the word a question uses to the key the records store it under, so the +# answer line is always keyed by the canonical field name. +FIELD_ALIASES = { + "department": "dept", + "dept": "dept", + "role": "role", + "hours": "hours", +} + +# The question forms this agent recognizes. +LOOKUP_RE = re.compile(r"what is the (\w+) of ([A-Za-z ]+)\?", re.IGNORECASE) +LIST_RE = re.compile(r"(?:list|how many) .*? in the (\w+) department", re.IGNORECASE) +COUNT_RE = re.compile(r"how many people are in the (\w+) department", re.IGNORECASE) + +# Never called. The address is unroutable so an accidental model call fails +# loudly rather than silently making the agent nondeterministic. +_DUMMY_LLM = CompletionClient(model="none", api_key="unused", api_base="http://127.0.0.1:1/v1") + + +class ReportAgent(Agent, llm=_DUMMY_LLM): + """Answer one question about the records file.""" + + _enable_tracing = True + + def __init__(self, **kwargs: object) -> None: + """Load the records the task environment supplied.""" + super().__init__(**kwargs) + self._records: list[dict] = json.loads(RECORDS_PATH.read_text(encoding="utf-8")) + + def solve(self, instruction: str) -> str: + """Return the single answer line this instruction asks for.""" + instruction = instruction[:MAX_INSTRUCTION_CHARS] + try: + for handler in (self.handle_lookup, self.handle_list, self.handle_count): + answer = handler(instruction) + if answer is not None: + return answer + except Exception: # noqa: BLE001 + # A handler fault must not take the process down: the caller still + # needs an answer line written, and a non-zero exit would be reported + # as a harness error rather than a scored result. + logger.exception("handler failed") + return FALLBACK + + def handle_lookup(self, instruction: str) -> str | None: + """Return `=` for the named record, or None if not a lookup.""" + match = LOOKUP_RE.search(instruction) + if match is None: + return None + field = FIELD_ALIASES.get(match.group(1).lower()) + if field is None: + return None + name = match.group(2).strip() + record = next(r for r in self._records if r["name"] == name) + return f"{field}={record[field]}" + + def handle_list(self, instruction: str) -> str | None: + """Return `names=` for a department, or None.""" + match = LIST_RE.search(instruction) + if match is None: + return None + dept = match.group(1).lower() + return "names=" + ",".join(r["name"] for r in self._records if r["dept"] == dept) + + def handle_count(self, instruction: str) -> str | None: + """Return `count=` for a department, or None.""" + match = COUNT_RE.search(instruction) + if match is None: + return None + dept = match.group(1).lower() + return f"count={sum(1 for r in self._records if r['dept'] == dept)}" diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py new file mode 100644 index 0000000000..38d6f54055 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor adapter for the smoke agent. + +No dependency install: NOOA is already in the task image. Harbor collects +/app/artifacts (declared by each task.toml) and /app/traces (injected by the +evaluator), so nothing is copied by hand. +""" + +from __future__ import annotations + +import fnmatch +import logging +import shlex +from pathlib import Path + +from harbor import AgentContext, BaseAgent, BaseEnvironment + +logger = logging.getLogger(__name__) + +AGENT_DIR = Path(__file__).parent + +# Never uploaded into the task container: optimizer bookkeeping, caches, and +# anything holding credentials. +# +# Most of these can no longer be siblings -- this file lives in the agent +# directory that `agent_source` points at, which holds the agent and nothing +# else. They are kept because the directory is a candidate workspace at run time +# and can accumulate caches, and because a credential file inside an agent +# directory must not travel regardless of layout. +EXCLUDE = { + "eval-and-optimize", + "__pycache__", + ".git", + ".claude", + ".uv", + ".venv", + ".env", + ".env.example", + "traces", + "artifacts", + "dataset", + "scripts", +} +EXCLUDE_GLOB = {"output.*", "*.md"} + + +class SymlinkedUploadError(RuntimeError): + """A selected upload path is, or contains, a symlink.""" + + +def _reject_symlinks(entries: list[Path]) -> None: + """Raise if any selected entry is a symlink or holds one at any depth. + + Args: + entries: Top-level paths selected for upload. + + Raises: + SymlinkedUploadError: naming the first offending path. + """ + for entry in entries: + offenders = [entry] if entry.is_symlink() else [] + if entry.is_dir() and not entry.is_symlink(): + offenders.extend(child for child in entry.rglob("*") if child.is_symlink()) + if offenders: + raise SymlinkedUploadError( + f"refusing to upload {offenders[0]}: it is a symlink, and following it would copy " + "host files outside the agent directory into the task container" + ) + + +class WrappedAgent(BaseAgent): + """Upload this agent directory into the container and run one task.""" + + @staticmethod + def name() -> str: + """Return the agent name Harbor records for a trial.""" + return "smoke-agent" + + def version(self) -> str | None: + """Return the agent version Harbor records for a trial.""" + return "1.0.0" + + async def setup(self, environment: BaseEnvironment) -> None: + """Upload the agent's source files. NOOA is already installed in the image.""" + selected = [ + entry + for entry in AGENT_DIR.iterdir() + if entry.name not in EXCLUDE and not any(fnmatch.fnmatch(entry.name, pattern) for pattern in EXCLUDE_GLOB) + ] + # Refuse symlinks before uploading anything. upload_dir follows them, so a + # link anywhere in a selected subtree would copy host files into the + # container. Scanned up front so a rejection cannot leave a half-populated + # /app. + _reject_symlinks(selected) + for entry in selected: + if entry.is_file(): + await environment.upload_file(entry, f"/app/{entry.name}") + elif entry.is_dir(): + await environment.upload_dir(entry, f"/app/{entry.name}") + logger.info("[setup] uploaded agent sources to /app") + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Execute the agent on *instruction* inside the task container.""" + session_id = self.session_id or "local" + proc = await environment.exec( + f"cd /app && python main.py --prompt {shlex.quote(instruction.strip())} " + f"--session-id {shlex.quote(session_id)}" + ) + + # Nothing is copied here: Harbor collects /app/artifacts and /app/traces + # after this method returns, per the declarations described above. + context.metadata = { + "stdout": proc.stdout, + "stderr": proc.stderr, + "returncode": proc.return_code, + } + if proc.return_code != 0: + raise RuntimeError(f"Agent process failed with exit code {proc.return_code}: {proc.stderr or proc.stdout}") diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py new file mode 100644 index 0000000000..2c98fcba9a --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Container entry point: solve one task and write the answer line.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from agent import ReportAgent + +ARTIFACTS_DIR = Path(os.environ.get("ARTIFACTS_DIR", "/app/artifacts")) +OUTPUT_PATH = ARTIFACTS_DIR / "output.txt" + + +def main() -> None: + """Run the agent on --prompt and write /app/artifacts/output.txt.""" + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", required=True) + parser.add_argument("--session-id", default=os.environ.get("HARBOR_SESSION_ID", "local")) + args = parser.parse_args() + + answer = ReportAgent().solve(args.prompt) + + ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH.write_text(answer + "\n", encoding="utf-8") + print(f"answer={answer!r} output={OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml new file mode 100644 index 0000000000..4204f3f9da --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Full scenario: several groups at once, multiple rounds, terminator decides when +# to stop. Point it at dataset/groups/_all. +# +# Two groups are held out of it -- one whose pass condition is the opposite of +# this one's, and one that is only reachable with trajectory scoring on, which it +# is not here. build_all_group.py has the evidence and the conditions for putting +# the second one back. Every task the combined set does contain is meant to be +# reachable, so a run that leaves one failing is a finding rather than an +# expected shortfall. +# +# This is the only scenario that exercises the evolutionary machinery -- survivors +# carried between rounds, ranking over more than two candidates, and the +# convergence check. The single-round configs leave all of it inert. +# +# EXPECT HOURS, NOT MINUTES. Roughly 25 min per round at three candidates: a bit +# over an hour if the terminator stops at three rounds, around two if it reaches +# the ceiling. That cost buys +# something tau-style benchmarks cannot: the expected outcome is known up front, +# so the result is checkable rather than merely plausible. + +# A ceiling, not a target. The terminator is expected to stop first; reaching the +# ceiling is itself a finding. +max_rounds: 5 + +# Default. The convergence check may not stop the run before this many rounds. +min_rounds_before_stopping: 3 + +max_survivors: 3 + +# Three per round. Enough for the Proposer to try genuinely different edits in a +# round -- past runs landed three separate improvements in one -- without paying for +# candidates that mostly duplicate each other. The Proposer may return fewer, and +# often does in later rounds; only an empty proposal is fatal. +max_candidates: 3 + +# Whole split. Six failing tasks and three that already pass, so there is a +# gradient to climb and a cost to breaking something that worked. +max_train_batch_tasks: null + +# Trajectory scoring is OFF, and that also switches off the goal tree it feeds. +# +# Not because it cannot work here. This comment used to say the scorer was +# defeated by the agent's determinism -- candidates indistinguishable, nothing to +# order by -- and that was wrong. The traces separate them plainly: the call +# graph names every method that ran and its status, so a candidate whose lookup +# errors is obvious beside one whose lookup returns a value. In a verification +# run the scorer ranked two candidates that were tied at 1.000 on reward, citing +# exactly that. +# +# What actually failed was the citation rule. Every score had to be grounded in +# span IDs, and the only sanctioned lookups are indexed by turn -- which an agent +# that makes no LLM calls never produces. The scorer could not satisfy its own +# contract, retried to the CodeAct ceiling, and raised GenerationError, which +# ended the run. Fixed in trace_scorer.py. +# +# It stays off because it is not yet dependable, and both failure modes are +# silent: a goal tree rejected on node count disables it for an entire run +# without saying so, and in the run that verified the fix it scored three +# candidates and returned nothing for the fourth, which then won. +# +# Consequence: trajectory scoring and the goal tree are NOT covered by this +# scenario. Multiple rounds, survivors, ranking, and the convergence check are. +# Turning it on is worthwhile once it is dependable -- build_all_group.py holds +# g5 out of the combined set for the same reason. +disable_trajectory_scoring: true + +# The one that matters here: the terminator deciding when to stop is the point. +disable_convergence_check: false + +evaluator: + n_attempts: 1 + n_concurrent_trials: 5 + quiet: true diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/single-round.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/single-round.yaml new file mode 100644 index 0000000000..89f558da12 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/single-round.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# One round, two candidates, whole train split. Both single-round scenarios use +# this file -- the repair gate and the generalization gate -- because the loop +# settings they want are identical. What separates them is the profile: +# +# optimizer.yaml -> g1-aggregation, the winner must beat baseline +# optimizer-generalization.yaml -> g4-dispatch-order, the baseline must be kept +# +# There were once two files here, differing in `max_train_batch_tasks`: the +# generalization scenario sampled a single train task on the theory that seeing +# one example made an over-narrow fix likely. That theory was wrong. What decides +# the scenario is how the splits are built -- whether train shows enough for a +# general fix to be reachable -- and sampling only dropped tasks, once excluding +# the control. Setting it back to `null` left two identical files, and a README +# claiming they differed. Merged rather than left as a trap. + +max_rounds: 1 +min_rounds_before_stopping: 1 +max_survivors: 1 + +# Two candidates so ranking and survivor selection do real work. With one, +# nothing is compared against anything and the two metric keys buy nothing. +max_candidates: 2 + +# Whole train split, for both scenarios. The Coder must see every kind of case +# the split carries, and the control must be present so a destructive fix costs +# reward. A number here silently drops tasks -- 2 excluded the control in a real +# run, leaving the round with nothing failing to analyze. +max_train_batch_tasks: null + +disable_trajectory_scoring: true +disable_convergence_check: true + +evaluator: + n_attempts: 1 + n_concurrent_trials: 3 + quiet: true diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile new file mode 100644 index 0000000000..a463c07589 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The whole task environment, built once and referenced by every task through +# [environment].docker_image. NOOA is baked in rather than installed during +# Harbor setup, so per-trial setup stays near zero and only a cached image build +# pays. The pinned revision must match the workspace root pyproject.toml; a test +# asserts it. + +FROM python:3.12-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv + +# `uv pip install --system` rather than `uv sync`: there is no project here, just +# one pinned dependency going into the image's own interpreter. +RUN uv pip install --system --no-cache \ + "nooa[tracing] @ git+https://github.com/NVIDIA-NeMo/labs-OO-Agents.git@6e0274dd03f883254a084cfb9f871ea580e03434" + +WORKDIR /app +RUN mkdir -p /app/artifacts /app/traces /app/data +COPY records.json /app/data/records.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json new file mode 100644 index 0000000000..10f5cad8e7 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json @@ -0,0 +1,8 @@ +[ + {"name": "Ada Lovelace", "dept": "research", "role": "engineer", "hours": 12}, + {"name": "Grace Hopper", "dept": "research", "role": "engineer", "hours": 8}, + {"name": "Zoë Washington", "dept": "research", "role": "analyst", "hours": 9}, + {"name": "O'Brien", "dept": "ops", "role": "operator", "hours": 5}, + {"name": "Ann-Marie Cruz", "dept": "ops", "role": "operator", "hours": 8}, + {"name": "Karl Jung", "dept": "ops", "role": "", "hours": 0} +] diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/instruction.md new file mode 100644 index 0000000000..43fe54662e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/task.toml new file mode 100644 index 0000000000..1697595e8c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-lookup-ada" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/lookup-ada/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/instruction.md new file mode 100644 index 0000000000..85e78ba658 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/instruction.md @@ -0,0 +1,9 @@ +What is the total hours for all engineers? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +total= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/task.toml new file mode 100644 index 0000000000..7c86956f13 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-total-hours-engineers" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/expected.txt new file mode 100644 index 0000000000..9484c7a746 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/expected.txt @@ -0,0 +1 @@ +total=20 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-engineers/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/instruction.md new file mode 100644 index 0000000000..a7053b6ea4 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/instruction.md @@ -0,0 +1,9 @@ +What is the total hours for the research department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +total= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/task.toml new file mode 100644 index 0000000000..24f82193b3 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-total-hours-research" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/expected.txt new file mode 100644 index 0000000000..d288905962 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/expected.txt @@ -0,0 +1 @@ +total=29 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/train/total-hours-research/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/instruction.md new file mode 100644 index 0000000000..3e1fa182f8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/instruction.md @@ -0,0 +1,9 @@ +What is the department of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/task.toml new file mode 100644 index 0000000000..b434620af3 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-lookup-grace" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/lookup-grace/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/instruction.md new file mode 100644 index 0000000000..3dc94e661c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/instruction.md @@ -0,0 +1,9 @@ +What is the total hours for all analysts? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +total= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/task.toml new file mode 100644 index 0000000000..0e3ddb693d --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-total-hours-analysts" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/expected.txt new file mode 100644 index 0000000000..2b7d3c0d3c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/expected.txt @@ -0,0 +1 @@ +total=9 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-analysts/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/instruction.md new file mode 100644 index 0000000000..18255009cf --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/instruction.md @@ -0,0 +1,9 @@ +What is the total hours for the ops department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +total= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/task.toml new file mode 100644 index 0000000000..b0ed1c036e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g1-total-hours-ops" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/expected.txt new file mode 100644 index 0000000000..f641e8371a --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/expected.txt @@ -0,0 +1 @@ +total=13 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g1-aggregation/validation/total-hours-ops/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/instruction.md new file mode 100644 index 0000000000..43fe54662e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/task.toml new file mode 100644 index 0000000000..b4b75bde0f --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-ada" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-ada/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/instruction.md new file mode 100644 index 0000000000..e07d469926 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/instruction.md @@ -0,0 +1,9 @@ +What is the department of O'Brien? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/task.toml new file mode 100644 index 0000000000..2c8b117e87 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-obrien" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/expected.txt new file mode 100644 index 0000000000..d7872bfad9 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/expected.txt @@ -0,0 +1 @@ +dept=ops diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-obrien/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/instruction.md new file mode 100644 index 0000000000..295a3ce830 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/instruction.md @@ -0,0 +1,9 @@ +What is the department of Zoë Washington? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/task.toml new file mode 100644 index 0000000000..5709892286 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-zoe" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/train/lookup-zoe/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/instruction.md new file mode 100644 index 0000000000..c992ec6f59 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ann-Marie Cruz? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/task.toml new file mode 100644 index 0000000000..d91a83118e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-ann-marie" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/expected.txt new file mode 100644 index 0000000000..d7872bfad9 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/expected.txt @@ -0,0 +1 @@ +dept=ops diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-ann-marie/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/instruction.md new file mode 100644 index 0000000000..3e1fa182f8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/instruction.md @@ -0,0 +1,9 @@ +What is the department of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/task.toml new file mode 100644 index 0000000000..65e7f74892 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-grace" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-grace/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/instruction.md new file mode 100644 index 0000000000..14ce49c7ec --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/instruction.md @@ -0,0 +1,9 @@ +What is the role of O'Brien? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +role= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/task.toml new file mode 100644 index 0000000000..4be11a5ca4 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g2-lookup-role-obrien" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g2"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/expected.txt new file mode 100644 index 0000000000..5880eaacdf --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/expected.txt @@ -0,0 +1 @@ +role=operator diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g2-name-patterns/validation/lookup-role-obrien/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/instruction.md new file mode 100644 index 0000000000..43fe54662e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/task.toml new file mode 100644 index 0000000000..dfb56a0a0e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-plain-dept" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/plain-dept/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/instruction.md new file mode 100644 index 0000000000..951f4b17dd --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/instruction.md @@ -0,0 +1,9 @@ +Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/task.toml new file mode 100644 index 0000000000..1101070c8d --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-preamble-dept" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-dept/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/instruction.md new file mode 100644 index 0000000000..e22e2c5fc9 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/instruction.md @@ -0,0 +1,9 @@ +Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the role of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +role= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/task.toml new file mode 100644 index 0000000000..47cdd290cf --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-preamble-role" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/expected.txt new file mode 100644 index 0000000000..abb47476ee --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/expected.txt @@ -0,0 +1 @@ +role=engineer diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/train/preamble-role/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/instruction.md new file mode 100644 index 0000000000..f3fbcf5ede --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/instruction.md @@ -0,0 +1,9 @@ +Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the hours of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +hours= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/task.toml new file mode 100644 index 0000000000..ae84e738e5 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-preamble-hours" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/expected.txt new file mode 100644 index 0000000000..a8650e892e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/expected.txt @@ -0,0 +1 @@ +hours=8 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-hours/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/instruction.md new file mode 100644 index 0000000000..57621b32d0 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/instruction.md @@ -0,0 +1,9 @@ +Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/task.toml new file mode 100644 index 0000000000..ee781489d3 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-preamble-long-dept" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/preamble-long-dept/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/instruction.md new file mode 100644 index 0000000000..437a614613 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/instruction.md @@ -0,0 +1,9 @@ +What is the department of Grace Hopper? Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/task.toml new file mode 100644 index 0000000000..5a86ff4c30 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g3-trailing-prose" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g3"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g3-long-inputs/validation/trailing-prose/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/instruction.md new file mode 100644 index 0000000000..01ae67c002 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/instruction.md @@ -0,0 +1,9 @@ +How many people are in the ops department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +count= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/task.toml new file mode 100644 index 0000000000..e6a5c44c16 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-count-ops" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/expected.txt new file mode 100644 index 0000000000..b4ed2d8eee --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/expected.txt @@ -0,0 +1 @@ +count=3 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-ops/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/instruction.md new file mode 100644 index 0000000000..8e611ac80d --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/instruction.md @@ -0,0 +1,9 @@ +How many people are in the research department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +count= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/task.toml new file mode 100644 index 0000000000..3967c69c87 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-count-research" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/expected.txt new file mode 100644 index 0000000000..b4ed2d8eee --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/expected.txt @@ -0,0 +1 @@ +count=3 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/count-research/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/instruction.md new file mode 100644 index 0000000000..43fe54662e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/task.toml new file mode 100644 index 0000000000..c1230cb51f --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-lookup-ada" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/train/lookup-ada/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/instruction.md new file mode 100644 index 0000000000..745afa18c8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/instruction.md @@ -0,0 +1,9 @@ +How many engineers are in the research department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +count= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/task.toml new file mode 100644 index 0000000000..be743073fe --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-count-engineers-research" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/expected.txt new file mode 100644 index 0000000000..8582bbcdea --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/expected.txt @@ -0,0 +1 @@ +count=2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-engineers-research/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/instruction.md new file mode 100644 index 0000000000..73f919bb06 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/instruction.md @@ -0,0 +1,9 @@ +How many operators are in the ops department? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +count= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/task.toml new file mode 100644 index 0000000000..be9e9f3e7a --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-count-operators-ops" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/expected.txt new file mode 100644 index 0000000000..8582bbcdea --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/expected.txt @@ -0,0 +1 @@ +count=2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/count-operators-ops/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/instruction.md new file mode 100644 index 0000000000..3e1fa182f8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/instruction.md @@ -0,0 +1,9 @@ +What is the department of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/task.toml new file mode 100644 index 0000000000..2202318bac --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g4-lookup-grace" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g4"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g4-dispatch-order/validation/lookup-grace/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/instruction.md new file mode 100644 index 0000000000..47667e7538 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/instruction.md @@ -0,0 +1,9 @@ +What is the role of Karl Jung? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +role= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/task.toml new file mode 100644 index 0000000000..621a26c32f --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-empty-role" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/expected.txt new file mode 100644 index 0000000000..752e7540d5 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/expected.txt @@ -0,0 +1 @@ +role=unknown diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/empty-role/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/instruction.md new file mode 100644 index 0000000000..43fe54662e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/instruction.md @@ -0,0 +1,9 @@ +What is the department of Ada Lovelace? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/task.toml new file mode 100644 index 0000000000..10f3b9f5c5 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-lookup-ada" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/lookup-ada/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/instruction.md new file mode 100644 index 0000000000..7df5c425b3 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/instruction.md @@ -0,0 +1,9 @@ +What is the department of Alan Turing? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/task.toml new file mode 100644 index 0000000000..ba7abcda87 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-missing-person" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/expected.txt new file mode 100644 index 0000000000..b3b94c8805 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/expected.txt @@ -0,0 +1 @@ +dept=unknown diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/train/missing-person/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/instruction.md new file mode 100644 index 0000000000..3e1fa182f8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/instruction.md @@ -0,0 +1,9 @@ +What is the department of Grace Hopper? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +dept= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/task.toml new file mode 100644 index 0000000000..26a4312029 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-lookup-grace" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/expected.txt new file mode 100644 index 0000000000..cd268d8615 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/expected.txt @@ -0,0 +1 @@ +dept=research diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/lookup-grace/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/instruction.md new file mode 100644 index 0000000000..4f3a39a176 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/instruction.md @@ -0,0 +1,9 @@ +What is the hours of Alan Turing? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +hours= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/task.toml new file mode 100644 index 0000000000..b5978c244c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-missing-person-hours" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/expected.txt new file mode 100644 index 0000000000..71e23732fd --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/expected.txt @@ -0,0 +1 @@ +hours=unknown diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-hours/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/instruction.md new file mode 100644 index 0000000000..d80499b2d5 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/instruction.md @@ -0,0 +1,9 @@ +What is the role of Alan Turing? + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` +role= +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/task.toml new file mode 100644 index 0000000000..fa6e052575 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/task.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/g5-missing-person-role" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g5"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. environment/ stays empty: Harbor +# requires the directory, and a Dockerfile would shadow the prebuilt image. +docker_image = "smoke-agent-env:sha-bc1849bfdcf5" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/expected.txt new file mode 100644 index 0000000000..752e7540d5 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/expected.txt @@ -0,0 +1 @@ +role=unknown diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/g5-edge-cases/validation/missing-person-role/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml new file mode 100644 index 0000000000..850a04196b --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# FROZEN ANALYST OUTPUT -- do not hand-edit. +# +# Generated 2026-08-05 by: +# nemo agents analyst run --workspace smoke-agent --agent smoke-agent \ +# --insights-file-output +# +# over three g1-aggregation/train traces recorded by scripts/record_traces.py: +# cb4eafc955233b5e249173ff16912b15 lookup-ada (passing, not referenced) +# bdb897d4701b43a5d2fabfe5320717dd total-hours-ops (failing) +# 9299f49567a3f79adc5a4e817e9e6a57 total-hours-research (failing) +# +# It is frozen so the analyst's nondeterminism stays out of a fixture that +# measures the Experimentalist. Regenerating means re-recording the traces +# first, since trace_refs must resolve in the target workspace. +# +# This file lives under dataset/ because _AGENT_COPY_EXCLUDE_NAMES excludes that +# directory: an Insight describes the weakness in prose, and the Coder must not +# read it. +insights: +- id: insight-632f7e00e110446db5b616ba25b4a50e + workspace: smoke-agent + title: Agent has no handler for 'total hours' sum-aggregation queries and returns + the 'I do not know' fallback + agent: smoke-agent + description: 'In the `g1-aggregation` evaluation, instructions of the form ''What + is the total hours for the department?'' (expecting a `total=` + line) are not answered. `solve` dispatches through its handler chain — `handle_lookup`, + `handle_list`, `handle_count` — none of which recognizes a request to SUM a numeric + field (hours) across the records for a department, so each returns None and `solve` + falls back to the literal string ''I do not know how to answer that.'' This is + written to `/app/artifacts/output.txt` instead of the required `total=` line, + failing the test case. Affected component: `agent.py` `solve`/handler dispatch + — there is a `handle_count` (record counting) but no `handle_sum`/total handler, + so any aggregation that requires summing a field is unsupported. Contrast with + single-record lookups (e.g. ''department of Ada Lovelace''), which succeed. Hypothesis: + the routing regexes and handlers cover lookup/list/count but omit sum/total aggregation, + so all ''total '' style questions fall through to the fallback. Observed + in test cases total-hours-ops and total-hours-research.' + status: open + trace_refs: + - bdb897d4701b43a5d2fabfe5320717dd + - 9299f49567a3f79adc5a4e817e9e6a57 + created_at: '2026-08-05T10:03:23.929608+00:00' + updated_at: '2026-08-05T10:03:23.929608+00:00' diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md new file mode 100644 index 0000000000..98853123f8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md @@ -0,0 +1,29 @@ + + + +# Task template + +Shape for tasks generated from an Insight's production traces. Three placeholders +are filled from the trace: `` and `` in `instruction.md`, and +`` in `tests/expected.txt`. + +## Constraints a generated task must respect + +- **`` must keep the grammar the agent already parses.** It recognises + exactly three forms, and a question outside them fails for the wrong reason -- + it looks like the weakness under test but is really a phrasing miss: + - `What is the of ?` + - `How many people are in the department?` + - `What is the total for the department?` +- **`` is the answer a *correct* agent would give**, keyed by the + canonical record field (`dept`, `role`, `hours`), never the word the question + used. The verifier compares the whole file byte-for-byte. +- **Do not edit `tests/test.sh`.** It is synced from `dataset/_shared/` and emits + the `reward` and `shape_ok` keys every task in a dataset must share. +- **Do not add `environment/Dockerfile`.** Tasks reference a prebuilt image via + `[environment].docker_image`; the empty `environment/` directory exists only + because Harbor requires the directory to be present. + +The records available to the agent are at `/app/data/records.json` in the image: +six people across the `research` and `ops` departments, with `name`, `dept`, +`role`, and `hours` fields. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md new file mode 100644 index 0000000000..9b116d66c6 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md @@ -0,0 +1,9 @@ + + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` += +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml new file mode 100644 index 0000000000..74c811fc9c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/generated" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-172a81e316b7" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt new file mode 100644 index 0000000000..288c2884a9 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt @@ -0,0 +1 @@ + diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh new file mode 100755 index 0000000000..c3c3e07496 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml new file mode 100644 index 0000000000..779be03544 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The generalization scenario's profile. Identical to optimizer.yaml except for +# the datasets, which is the whole point: what separates a repair scenario from a +# generalization one is the split, not any tuning knob. +# +# nemo agents experimentalist run --profile optimizer-generalization.yaml \ +# --no-insight --config configs/single-round.yaml +# +# g4-dispatch-order's train split shows only plain department counts, so the +# reachable fix -- reordering the handler chain -- passes train. Validation asks +# for role-scoped counts, which that fix does not reach. A healthy run therefore +# *keeps the baseline*, the opposite of what the repair scenario asserts. +# +# This file exists because a config alone could not express that. Both scenario +# configs are identical, and optimizer.yaml pins the datasets to g1-aggregation, +# so pointing --config at generalization.yaml ran the repair scenario against +# g1 and quietly answered the wrong question. + +# Logical agent name. In insight mode it must match the Insight's agent; under +# --no-insight it is only a label. +agent: smoke-agent + +# Same agent under test as the repair scenario. Only the tasks differ. +agent_source: ./agent + +agent_spec: ./AGENT-SPEC.md + +# Required by the profile schema even under --no-insight, where it is never read. +task_template: ./dataset/task-template + +datasets: + train: ./dataset/groups/g4-dispatch-order/train + validation: ./dataset/groups/g4-dispatch-order/validation + +workspace: default diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml new file mode 100644 index 0000000000..4a3b376343 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Smoke fixture profile. Everything is local: a "./" prefix classifies as a local +# path, so no registry_url is needed and nothing is downloaded. +# +# nemo agents experimentalist doctor --profile +# nemo agents experimentalist run --profile --no-insight + +# Logical agent name. In insight mode it must match the Insight's agent; under +# --no-insight it is only a label. +agent: smoke-agent + +# Where the baseline agent's code lives, relative to this file. The whole +# directory is copied into every candidate workspace and is readable by the +# Coder, so it holds the agent and nothing else: no README, no configs, no +# scripts, no dataset, no .env. Anything that should not reach the Coder simply +# stays out of this folder rather than relying on an exclusion list. +agent_source: ./agent + +# Behaviour contract threaded to the analyzer and goal-tree components. It states +# the rules the agent must keep, chiefly that it stays offline and reproducible. +agent_spec: ./AGENT-SPEC.md + +# Required by the profile schema even under --no-insight, where it is never read. +# It points at a single Harbor task directory used as the shape for Eval Author +# generated tasks in insight mode. +task_template: ./dataset/task-template + +datasets: + train: ./dataset/groups/g1-aggregation/train + validation: ./dataset/groups/g1-aggregation/validation + +workspace: default diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py new file mode 100644 index 0000000000..6f4de71e02 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assemble `dataset/groups/_all/` from the groups listed in `source_groups`. + +The loop takes one train/validation pair, so exercising several groups in a +single run means one combined dataset. Directory names collide -- every group has +a `lookup-ada` or `lookup-grace` control -- so each is prefixed with its group +key. The `[task] name` values inside were already unique. + +Generated, not authored: rerun this after changing any group. A test asserts the +combined copy matches its sources, so a stale `_all/` fails rather than silently +running against old tasks. +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +COMBINED = "_all" +SPLITS = ("train", "validation") + +# Not part of the combined set. Both are held out because the combined scenario +# asserts that every task in it is reachable, and neither of these is. +# +# `g4-dispatch-order` backs the generalization scenario, which asserts the +# opposite outcome: there, retaining the baseline is a pass. One run cannot +# assert both. +# +# `g5-edge-cases` is reachable only when trajectory scoring is on, and the +# combined scenario runs with it off. Measured across runs made after the spec +# stated the sentinel, so the spec is not the variable: +# +# goal tree off -> 0 of 13 candidates closed g5 +# goal tree on -> 7 of 11 candidates closed g5 +# +# The pattern is that the goal tree sharpens the analysis enough for the Coder to +# see both halves of the fix; without it the analysis names one half and the +# candidates fix one half, which scores nothing. Trajectory scoring is off here +# because it is not yet dependable -- it has silently skipped a candidate, and a +# rejected goal tree disables it for a whole run without saying so. +# +# PUT G5 BACK once trajectory scoring is dependable enough to leave on. It is the +# only group that exercises a fix needing several coordinated edits, so the +# combined scenario is weaker without it. +EXCLUDED_GROUPS = frozenset({"g4-dispatch-order", "g5-edge-cases"}) + + +def group_key(group: str) -> str: + """Return the short key a group's task names already use (``g1`` from ``g1-aggregation``).""" + return group.split("-")[0] + + +def source_groups(dataset_dir: Path) -> list[str]: + """Every group the combined set is built from, in a stable order.""" + groups = dataset_dir / "groups" + return sorted( + d.name for d in groups.iterdir() if d.is_dir() and d.name != COMBINED and d.name not in EXCLUDED_GROUPS + ) + + +def assemble(dataset_dir: Path) -> list[Path]: + """Rebuild the combined group from its sources; return the task directories written.""" + target = dataset_dir / "groups" / COMBINED + shutil.rmtree(target, ignore_errors=True) + + written: list[Path] = [] + for group in source_groups(dataset_dir): + key = group_key(group) + for split in SPLITS: + for task in sorted((dataset_dir / "groups" / group / split).iterdir()): + if not (task / "task.toml").is_file(): + continue + dest = target / split / f"{key}-{task.name}" + shutil.copytree(task, dest) + written.append(dest) + return written + + +def main() -> None: + """Assemble the combined group from the command line.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + args = parser.parse_args() + written = assemble(args.dataset_dir) + for path in written: + print(path.relative_to(args.dataset_dir)) + print(f"{len(written)} tasks") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py new file mode 100644 index 0000000000..4edd4f5e09 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the shared task image and stamp its tag into every task.toml. + +The tag is a content hash of the Dockerfile and the records file, so a change to +either produces a new tag. Tasks reference the tag rather than carrying their own +Dockerfile, and a test asserts every task references the current one -- which is +what turns "forgot to rebuild" into a failing test instead of a container quietly +running against stale data. +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +from pathlib import Path + +HASHED_FILES = ("Dockerfile", "records.json") +IMAGE_NAME = "smoke-agent-env" +_DOCKER_IMAGE_RE = re.compile(r'^(docker_image\s*=\s*)"[^"]*"', re.MULTILINE) + + +def image_tag(shared_dir: Path) -> str: + """Return the content-addressed tag for the current shared assets.""" + digest = hashlib.sha256() + for name in HASHED_FILES: + digest.update((shared_dir / name).read_bytes()) + return f"{IMAGE_NAME}:sha-{digest.hexdigest()[:12]}" + + +def build(shared_dir: Path, tag: str) -> None: + """Build the image. Docker layer caching makes a no-op rebuild cheap.""" + subprocess.run(["docker", "build", "-t", tag, str(shared_dir)], check=True) + + +def ensure_environment_dirs(dataset_dir: Path) -> list[Path]: + """Create the empty environment/ every task needs; return the ones created. + + Harbor's ``TaskModel.is_valid_dir`` requires ``environment/`` to *exist* before + it will even parse a task; ``[environment].docker_image`` only makes the + Dockerfile inside it optional. A task without the directory is silently not a + task -- the dataset loads with zero tasks rather than erroring. The directory + stays empty apart from a .gitkeep, since a Dockerfile there would shadow the + prebuilt image. + """ + created: list[Path] = [] + for task_toml in sorted((dataset_dir / "groups").rglob("task.toml")): + keep = task_toml.parent / "environment" / ".gitkeep" + if not keep.exists(): + keep.parent.mkdir(parents=True, exist_ok=True) + keep.touch() + created.append(keep) + return created + + +def stamp_tasks(dataset_dir: Path, tag: str) -> list[Path]: + """Rewrite every task.toml's docker_image to *tag*; return the ones changed.""" + changed: list[Path] = [] + for task_toml in sorted((dataset_dir / "groups").rglob("task.toml")): + text = task_toml.read_text(encoding="utf-8") + updated = _DOCKER_IMAGE_RE.sub(rf'\1"{tag}"', text) + if updated != text: + task_toml.write_text(updated, encoding="utf-8") + changed.append(task_toml) + return changed + + +def main() -> None: + """Build the shared image and stamp its tag into every task.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + parser.add_argument("--skip-build", action="store_true", help="stamp tags without invoking docker") + args = parser.parse_args() + + tag = image_tag(args.dataset_dir / "_shared") + if not args.skip_build: + build(args.dataset_dir / "_shared", tag) + for path in ensure_environment_dirs(args.dataset_dir): + print(f"created {path}") + for path in stamp_tasks(args.dataset_dir, tag): + print(f"stamped {path}") + print(tag) + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py new file mode 100644 index 0000000000..4e9821db76 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Evaluate one group's tasks and upload their traces to Intake. + +Insight mode needs an Insight whose trace_refs resolve through the Platform +client, so a group's own tasks double as the trace source: evaluate them, then +ingest the trace each trial wrote. + +The resource attributes attached here are not decoration. `gen_ai.agent.name` is +how the analyst later finds these traces, so the Insight generated from them is +only possible because they are set. +""" + +from __future__ import annotations + +import argparse +import asyncio +from pathlib import Path + +from nemo_experimentalist_plugin.client import make_client +from nemo_experimentalist_plugin.entities import TrialResult, local_path_from_uri +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import ( + HarborDataset, + HarborEvaluator, + HarborEvaluatorConfig, +) +from nemo_experimentalist_plugin.experimentalist.otlp import jsonl_to_protobuf, read_trace_id +from nemo_platform import AsyncNeMoPlatform, NotFoundError + +AGENT_NAME = "smoke-agent" +AGENT_VERSION = "1.0.0" +INGEST_PATH = "/apis/intake/v2/workspaces/{workspace}/ingest/otlp/v1/traces" +POLL_ATTEMPTS = 30 +POLL_DELAY_SECONDS = 2.0 + + +async def _upload_trials( + client: AsyncNeMoPlatform, + trials: list[TrialResult], + *, + workspace: str, + group: str, +) -> dict[str, str]: + """Upload each trial's trace; return {trial_id: trace_id}.""" + url = INGEST_PATH.format(workspace=workspace) + trace_ids: dict[str, str] = {} + + for trial in trials: + if trial.trace is None: + raise RuntimeError(f"Trial {trial.id} produced no trace; the agent must write /app/traces") + trace_id = read_trace_id(trial.trace) + if trace_id in trace_ids.values(): + raise RuntimeError(f"Trace {trace_id} was produced by more than one trial") + + attrs = { + "nemo.experiment.id": group, + "nemo.test_case.id": trial.task_id, + "nemo.trial.id": trial.id, + "gen_ai.agent.name": AGENT_NAME, + "gen_ai.agent.version": AGENT_VERSION, + } + path = local_path_from_uri(trial.trace.uri, context="Agent execution trace") + payloads = jsonl_to_protobuf(path, extra_resource_attrs=attrs) + if not payloads: + raise RuntimeError(f"Trial {trial.id} produced an empty trace") + for payload in payloads: + await client.post( + url, + cast_to=object, + content=payload, + options={"headers": {"Content-Type": "application/x-protobuf"}}, + ) + trace_ids[trial.id] = trace_id + + return trace_ids + + +async def _wait_retrievable(client: AsyncNeMoPlatform, workspace: str, trace_ids: set[str]) -> None: + """Block until every trace id resolves, or raise once the budget is spent.""" + pending = set(trace_ids) + for _ in range(POLL_ATTEMPTS): + for trace_id in sorted(pending): + try: + await client.intake.traces.retrieve(trace_id, workspace=workspace) + except NotFoundError: + continue + pending.discard(trace_id) + if not pending: + return + await asyncio.sleep(POLL_DELAY_SECONDS) + raise TimeoutError(f"traces never became retrievable: {sorted(pending)}") + + +async def run(args: argparse.Namespace) -> dict[str, str]: + """Evaluate the group's train split, then upload every trial's trace.""" + agent_path = args.agent.expanduser().resolve() + dataset_path = (args.dataset_root / "groups" / args.group / args.split).expanduser().resolve() + if not dataset_path.is_dir(): + raise FileNotFoundError(f"group dataset not found: {dataset_path}") + + dataset = HarborDataset.from_path(dataset_path) + run_dir = args.output.expanduser().resolve() / args.group / args.split + if run_dir.exists(): + raise FileExistsError(f"output directory already exists: {run_dir}") + run_dir.mkdir(parents=True) + + client = make_client(args.base_url) + try: + await client.workspaces.create( + name=args.workspace, + description="smoke-agent traces for Insights", + exist_ok=True, + ) + options = HarborEvaluatorConfig( + job_name=f"smoke-{args.group}-{args.split}-record", + jobs_dir=Path("results"), + n_attempts=1, + n_concurrent_trials=args.concurrency, + quiet=True, + ) + result = await HarborEvaluator(experiment_dir=run_dir).run( + agent=agent_path, + dataset=dataset, + options=options, + ) + trials = list(result.trials) + if not trials: + raise RuntimeError(f"no trials produced under {run_dir}") + + trace_ids = await _upload_trials(client, trials, workspace=args.workspace, group=args.group) + await _wait_retrievable(client, args.workspace, set(trace_ids.values())) + return trace_ids + finally: + await client.close() + + +def main() -> None: + """Record and ingest one group's traces.""" + example_dir = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser() + parser.add_argument("--group", required=True) + parser.add_argument("--split", default="train") + parser.add_argument("--workspace", default="smoke-agent") + parser.add_argument("--agent", type=Path, default=example_dir) + parser.add_argument("--dataset-root", type=Path, default=example_dir / "dataset") + parser.add_argument("--output", type=Path, default=Path("tmp/smoke-record")) + parser.add_argument("--base-url", default=None) + parser.add_argument("--concurrency", type=int, default=3) + args = parser.parse_args() + + for trial_id, trace_id in asyncio.run(run(args)).items(): + print(f"{trial_id} {trace_id}") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/sync_verifier.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/sync_verifier.py new file mode 100644 index 0000000000..de96b07c6e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/sync_verifier.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Copy dataset/_shared/test.sh into every task's tests/ directory. + +Harbor copies each task's tests/ into the container at verify time alongside that +task's own expected.txt, so the verifier is necessarily duplicated per task even +though it is identical everywhere. This is the one place that duplication is +produced; test_smoke_agent_assets.py keeps it honest. + +The Dockerfile and records file are *not* copied. Tasks reference the prebuilt +image by tag instead — see scripts/build_image.py. +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + + +def sync(dataset_dir: Path) -> list[Path]: + """Copy the canonical verifier into every task; return the paths written.""" + canonical = dataset_dir / "_shared" / "test.sh" + written: list[Path] = [] + for task_toml in sorted((dataset_dir / "groups").rglob("task.toml")): + target = task_toml.parent / "tests" / "test.sh" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(canonical, target) + target.chmod(0o755) + written.append(target) + return written + + +def main() -> None: + """Sync the verifier from the command line.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + args = parser.parse_args() + for path in sync(args.dataset_dir): + print(path) + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py new file mode 100644 index 0000000000..b9e709dc1a --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-weakness unit tests for the smoke agent. No Docker, no network. + +Each test pins one documented behaviour of the baseline agent. A failure here +usually means someone "fixed" the agent; see +plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md first. +""" + +from __future__ import annotations + +import importlib.util +import os +import re +import sys +from pathlib import Path +from typing import Any + +import pytest +from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import _AGENT_COPY_EXCLUDE_NAMES + +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_RECORDS = _EXAMPLE_DIR / "dataset" / "_shared" / "records.json" + + +@pytest.fixture(scope="module") +def agent_module(tmp_path_factory: pytest.TempPathFactory) -> Any: + """Import agent.py by path; it is not an installed package.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ["TRACE_DIR"] = str(tmp_path_factory.mktemp("traces")) + spec = importlib.util.spec_from_file_location("_smoke_agent", _EXAMPLE_DIR / "agent" / "agent.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module + + +def test_working_lookup_succeeds(agent_module: Any) -> None: + """The control path: plain ASCII names resolve, so controls pass at baseline.""" + agent = agent_module.ReportAgent() + assert agent.solve("What is the department of Ada Lovelace?") == "dept=research" + + +def test_g1_no_aggregation_capability(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + answer = agent.solve("What is the total hours for the research department?") + assert answer == agent_module.FALLBACK + + +def test_g2_punctuated_names_fall_through(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + for name in ("O'Brien", "Zoë Washington", "Ann-Marie Cruz"): + assert agent.solve(f"What is the department of {name}?") == agent_module.FALLBACK + + +def test_g3_long_instruction_is_clipped(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + preamble = "Reporting policy applies to this request. " * 10 + assert len(preamble) > agent_module.MAX_INSTRUCTION_CHARS + question = "What is the department of Grace Hopper?" + assert agent.solve(question) == "dept=research" + assert agent.solve(preamble + question) == agent_module.FALLBACK + + +def test_g4_list_handler_shadows_count(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + answer = agent.solve("How many people are in the research department?") + assert answer.startswith("names="), "expected the greedy list handler to win" + assert answer != "count=3" + + +def test_g5_missing_record_does_not_degrade(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + assert agent.solve("What is the department of Alan Turing?") == agent_module.FALLBACK + + +def test_g5_empty_field_does_not_degrade(agent_module: Any) -> None: + agent = agent_module.ReportAgent() + assert agent.solve("What is the role of Karl Jung?") == "role=" + + +def test_agent_is_deterministic(agent_module: Any) -> None: + """Repeated identical input must give byte-identical output.""" + agent = agent_module.ReportAgent() + question = "What is the department of Ada Lovelace?" + assert len({agent.solve(question) for _ in range(20)}) == 1 + + +def test_agent_declares_no_strategy_methods() -> None: + """A @strategy method would make the agent LLM-driven and nondeterministic.""" + source = (_EXAMPLE_DIR / "agent" / "agent.py").read_text(encoding="utf-8") + assert "@strategy" not in source + assert "CodeActStrategy" not in source + + +def test_spec_forbids_llm_backed_changes() -> None: + """The Coder reads AGENT-SPEC.md first; the determinism rule has to be in it.""" + spec = (_EXAMPLE_DIR / "AGENT-SPEC.md").read_text(encoding="utf-8").lower() + for phrase in ("@strategy", "deterministic", "no llm", "offline"): + assert phrase in spec, f"AGENT-SPEC.md must mention {phrase!r}" + + +# Names for what the fixture measures. +_LEAK_TERMS = ("weakness", "known gap", "deliberate", "on purpose", "do not fix") + +# Only what `agent_source` points at is copied into a candidate workspace, minus +# whatever the copier drops on the way. Imported rather than restated: a +# hand-written skip list drifts, and it drifted before -- an earlier version +# skipped `traces`, which the copier does *not* exclude, so a description left +# there would have reached the Coder without failing this guard. +_AGENT_SOURCE_DIRNAME = "agent" +_LEAK_SCAN_SKIP_DIRS = _AGENT_COPY_EXCLUDE_NAMES + + +def test_agent_source_points_at_the_agent_subdirectory() -> None: + """The leak scan below only covers `agent/`, which is only sound while this holds. + + Widening `agent_source` back to `.` would put README, configs, and scripts + back in front of the Coder while the scan still checked one subdirectory -- + the failure would be silent, which is exactly the shape of bug that put a + live `.env` into every candidate workspace. + """ + profile = (_EXAMPLE_DIR / "optimizer.yaml").read_text(encoding="utf-8") + found = re.search(r"^agent_source:\s*(\S+)", profile, re.MULTILINE) + assert found is not None, "optimizer.yaml must declare agent_source" + assert found.group(1) == f"./{_AGENT_SOURCE_DIRNAME}", ( + f"agent_source is {found.group(1)!r}; the leak scan assumes ./{_AGENT_SOURCE_DIRNAME}" + ) + + +def test_agent_source_does_not_leak_the_weaknesses() -> None: + """Nothing the Coder can read may describe what the fixture measures. + + A description there hands the Coder the diagnosis this fixture exists to + test, and a run could pass with a broken Analyzer. The boundary is structural + rather than a list of exclusions: material that explains the fixture lives + outside `agent/`, so README, configs, and scripts are free to be candid. + + AGENT-SPEC.md reaches the LLM components by a separate path and is covered by + test_spec_does_not_leak_the_weaknesses. + """ + agent_dir = _EXAMPLE_DIR / _AGENT_SOURCE_DIRNAME + offenders: list[str] = [] + for path in sorted(agent_dir.rglob("*")): + if not path.is_file() or set(path.relative_to(agent_dir).parts) & _LEAK_SCAN_SKIP_DIRS: + continue + try: + text = path.read_text(encoding="utf-8").lower() + except (UnicodeDecodeError, OSError): + continue + offenders.extend(f"{path.relative_to(agent_dir)}: {term!r}" for term in _LEAK_TERMS if term in text) + assert not offenders, "agent source leaks what the fixture measures:\n " + "\n ".join(offenders) + + +def test_spec_does_not_leak_the_weaknesses() -> None: + """The spec is copied into source-agent/ and read before any source file. + + Naming the gaps there hands the Coder the diagnosis the fixture exists to + test, and lets a run pass with a broken Analyzer. + """ + spec = (_EXAMPLE_DIR / "AGENT-SPEC.md").read_text(encoding="utf-8").lower() + for leak in ("weakness", "known gap", "deliberate", "aggregat", "on purpose"): + assert leak not in spec, f"AGENT-SPEC.md must not mention {leak!r}" diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py new file mode 100644 index 0000000000..86d9e9df85 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the smoke agent's task image, its NOOA pin, and its verifier honest. + +No Docker here on purpose: the image tag is a content hash, so a forgotten +rebuild is a string comparison rather than something only a container run can +reveal. +""" + +from __future__ import annotations + +import functools +import hashlib +import importlib.util +import re +import tomllib +from pathlib import Path +from typing import Any + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_SHARED = _EXAMPLE_DIR / "dataset" / "_shared" +_HASHED = ("Dockerfile", "records.json") + + +def _root_nooa_rev() -> str: + data = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + return data["tool"]["uv"]["sources"]["nooa"]["rev"] + + +def _expected_tag() -> str: + digest = hashlib.sha256() + for name in _HASHED: + digest.update((_SHARED / name).read_bytes()) + return f"smoke-agent-env:sha-{digest.hexdigest()[:12]}" + + +def _task_tomls() -> list[Path]: + groups = _EXAMPLE_DIR / "dataset" / "groups" + return sorted(groups.rglob("task.toml")) if groups.is_dir() else [] + + +_EXPECTED_METRIC_KEYS = ("reward", "shape_ok") + + +def test_verifier_emits_exactly_the_two_metric_keys() -> None: + """The Experimentalist averages metrics across trials and rejects inconsistent sets. + + Asserting only that both names appear somewhere would also accept a third + key, or a second write, either of which changes the metric set every trial + reports. Parse what is actually emitted instead. + """ + code = _verifier_code() + writes = [line.strip() for line in code.splitlines() if "reward.json" in line and "printf" in line] + assert len(writes) == 1, f"expected exactly one line writing reward.json, found {len(writes)}: {writes}" + keys = tuple(re.findall(r'"([A-Za-z_][A-Za-z0-9_]*)"\s*:', writes[0])) + assert keys == _EXPECTED_METRIC_KEYS, f"verifier emits {keys}, expected exactly {_EXPECTED_METRIC_KEYS}" + + +def _verifier_code() -> str: + """Return test.sh with comment lines dropped. + + Every guard below is a substring check, and the script documents each choice + in a comment that names the rejected alternative. Checking raw text would + match those comments rather than the code. + """ + lines = (_SHARED / "test.sh").read_text(encoding="utf-8").splitlines() + return "\n".join(line for line in lines if not line.lstrip().startswith("#")) + + +def _shell_options(code: str) -> tuple[set[str], set[str]]: + """Return (short flags, `-o` long options) enabled across every `set` line. + + Parsed rather than string-matched so the contract holds however it is spelled: + ``set -uo pipefail``, ``set -u -o pipefail`` and ``set -euo pipefail`` all + resolve to the same options. + """ + short: set[str] = set() + long: set[str] = set() + for raw in code.splitlines(): + line = raw.strip() + if not line.startswith("set "): + continue + tokens = line.split()[1:] + index = 0 + while index < len(tokens): + token = tokens[index] + index += 1 + if not token.startswith("-") or token.startswith("--"): + continue + for flag in token[1:]: + if flag == "o" and index < len(tokens): + long.add(tokens[index]) + index += 1 + else: + short.add(flag) + return short, long + + +def test_verifier_sets_exactly_the_intended_shell_options() -> None: + """`set -uo pipefail` is the contract: nounset and pipefail on, errexit off. + + Checking only that errexit is absent would also pass a verifier with no + `set` options at all. Both of the others earn their place: without nounset an + unset variable expands to empty and a comparison can succeed against nothing, + and without pipefail a failing stage of a pipeline is invisible. + """ + short, long = _shell_options(_verifier_code()) + assert "u" in short, "nounset is off; an unset variable expands to empty and can score a wrong answer" + assert "pipefail" in long, "pipefail is off; a failing stage of a pipeline would go unnoticed" + assert "e" not in short, ( + "errexit is on; aborting before reward.json is written turns a legitimate 0 into a missing metric" + ) + + +def test_verifier_keeps_its_reward_hacking_guards() -> None: + """Each of these was a deliberate choice; losing one reopens a way to score without solving.""" + code = _verifier_code() + + assert "tr -d" not in code, "tr -d '\\r' deletes every CR, collapsing sum=42 into a passing sum=42" + assert "cmp -s" in code, "whole-file compare; command substitution strips trailing newlines" + assert "refusing to score" in code, "must fail closed when the expected fixture is unreadable" + + +def test_verifier_does_not_echo_answers() -> None: + """Expected/actual values in the verifier log would let a candidate hardcode them.""" + code = _verifier_code() + for forbidden in ('cat "$EXPECTED_NORM"', 'cat "$ACTUAL_NORM"'): + assert forbidden not in code, f"{forbidden} publishes ground truth to the trial log" + + +def test_dockerfile_nooa_rev_matches_workspace() -> None: + """A second copy of the revision string; drift means the fixture tests a different NOOA.""" + dockerfile_path = _SHARED / "Dockerfile" + if not dockerfile_path.is_file(): + return # Task 4 creates it. + found = re.search(r"labs-OO-Agents\.git@([0-9a-f]{40})", dockerfile_path.read_text(encoding="utf-8")) + assert found is not None, "Dockerfile must pin NOOA to an explicit revision" + assert found.group(1) == _root_nooa_rev() + + +def test_every_task_references_the_current_image() -> None: + """A stale tag means containers run against different records than the repo has.""" + tasks = _task_tomls() + if not tasks: + return # Task 6 creates them. + expected = _expected_tag() + for task_toml in tasks: + actual = tomllib.loads(task_toml.read_text(encoding="utf-8"))["environment"]["docker_image"] + assert actual == expected, ( + f"{task_toml.parent.name} references {actual}, current content is {expected}. " + "Run scripts/build_image.py after changing the Dockerfile or records.json." + ) + + +def test_every_task_carries_the_current_verifier() -> None: + """Harbor copies tests/ per task, so test.sh is duplicated; keep the copies identical.""" + tasks = _task_tomls() + if not tasks: + return # Task 6 creates them. + canonical = (_SHARED / "test.sh").read_bytes() + for task_toml in tasks: + actual = (task_toml.parent / "tests" / "test.sh").read_bytes() + assert actual == canonical, f"{task_toml.parent.name}/tests/test.sh is stale; run scripts/sync_verifier.py" + + +def test_every_task_has_an_empty_environment_dir() -> None: + """Harbor requires environment/ to exist, but it must stay empty here. + + ``TaskModel.is_valid_dir`` returns False when environment/ is absent, and a + dataset whose tasks all fail that check loads with *zero tasks* rather than + raising -- so a missing directory looks like an empty dataset, not an error. + ``[environment].docker_image`` only makes the Dockerfile inside it optional. + + A Dockerfile there would shadow the prebuilt image and reintroduce the + per-task build the content-hash tag exists to avoid. + """ + for task_toml in _task_tomls(): + environment = task_toml.parent / "environment" + assert environment.is_dir(), ( + f"{task_toml.parent.name} has no environment/; Harbor will not see it as a task. " + "Run scripts/build_image.py." + ) + contents = {p.name for p in environment.iterdir()} - {".gitkeep"} + assert not contents, f"{task_toml.parent.name}/environment must stay empty, found {sorted(contents)}" + + +@functools.cache +def _builder() -> Any: + """Import scripts/build_all_group.py by path; scripts/ is not a package. + + The exclusion list lives there, so the test reads it rather than repeating + it -- a second copy would drift the moment a group is added or removed. + """ + path = _EXAMPLE_DIR / "scripts" / "build_all_group.py" + spec = importlib.util.spec_from_file_location("_smoke_build_all_group", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_excluded_groups_stay_out_of_the_combined_set() -> None: + """A group whose scenario expects the baseline to win cannot share a run with one that expects a fix. + + Combining them leaves the full scenario with no reachable pass criterion, so + an accidental re-inclusion has to fail loudly rather than just lower the score. + """ + combined = _EXAMPLE_DIR / "dataset" / "groups" / "_all" + if not combined.is_dir(): + return # optional; only the full scenario needs it + + excluded_keys = {name.split("-")[0] for name in _builder().EXCLUDED_GROUPS} + assert excluded_keys, "the exclusion list should not be empty; see build_all_group.py" + present = {task.parent.name.split("-")[0] for task in combined.rglob("task.toml")} + assert not (present & excluded_keys), ( + f"combined group contains excluded group(s) {sorted(present & excluded_keys)}; run scripts/build_all_group.py" + ) + + +def test_combined_group_matches_its_sources() -> None: + """`_all` is generated from the other groups; a stale copy runs old tasks silently. + + Rebuild with scripts/build_all_group.py after changing any group. + """ + groups = _EXAMPLE_DIR / "dataset" / "groups" + combined = groups / "_all" + if not combined.is_dir(): + return # optional; only the full scenario needs it + + builder = _builder() + expected: dict[str, Path] = {} + for group_name in builder.source_groups(_EXAMPLE_DIR / "dataset"): + group = groups / group_name + key = builder.group_key(group_name) + for split in ("train", "validation"): + for task in sorted((group / split).iterdir()): + if (task / "task.toml").is_file(): + expected[f"{split}/{key}-{task.name}"] = task + + actual = {f"{t.parent.parent.name}/{t.parent.name}": t.parent for t in combined.rglob("task.toml")} + assert set(actual) == set(expected), ( + "combined group is stale; run scripts/build_all_group.py. " + f"missing={sorted(set(expected) - set(actual))} unexpected={sorted(set(actual) - set(expected))}" + ) + for rel, src in expected.items(): + for name in ("instruction.md", "task.toml", "tests/expected.txt"): + assert (actual[rel] / name).read_bytes() == (src / name).read_bytes(), ( + f"combined {rel}/{name} differs from its source; run scripts/build_all_group.py" + ) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py new file mode 100644 index 0000000000..3477e72e9f --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Guard the smoke agent's deliberately weak baseline and its record set. + +The fixture only measures anything because the agent ships with known +weaknesses. A well-meaning edit that closes one silently destroys what an +Experimentalist run is asserted against, so the expected baseline is pinned +here. These tests need no Docker and no network. + +See plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md before changing +either the agent or this file. +""" + +from __future__ import annotations + +import functools +import importlib.util +import json +import os +import re +import sys +import tempfile +from pathlib import Path +from typing import Any + +import pytest + +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_RECORDS = _EXAMPLE_DIR / "dataset" / "_shared" / "records.json" + + +def _records() -> list[dict]: + return json.loads(_RECORDS.read_text(encoding="utf-8")) + + +def test_department_totals_are_pinned() -> None: + """G1's expected outputs are derived from these sums; drift breaks every G1 task.""" + totals: dict[str, int] = {} + for record in _records(): + totals[record["dept"]] = totals.get(record["dept"], 0) + record["hours"] + assert totals == {"research": 29, "ops": 13} + assert sum(totals.values()) == 42 + + +def test_role_scoped_hours_are_pinned() -> None: + """G1 scopes sums by role as well as by department, in both splits.""" + by_role: dict[str, int] = {} + for record in _records(): + by_role[record["role"]] = by_role.get(record["role"], 0) + record["hours"] + assert by_role["engineer"] == 20 # train + assert by_role["analyst"] == 9 # validation + + +def test_g2_names_carry_no_other_weakness() -> None: + """G2 needs punctuated / non-ASCII names, and those records must be otherwise clean. + + The predicate mirrors the character class the agent's lookup pattern accepts. + `str.isalpha()` is deliberately not used: it is Unicode-aware, so "Zoë" passes + it while the agent's ASCII-only class rejects the name. + """ + tricky = [r for r in _records() if not re.fullmatch(r"[A-Za-z ]+", r["name"])] + assert {r["name"] for r in tricky} == {"O'Brien", "Zoë Washington", "Ann-Marie Cruz"} + for record in tricky: + assert record["role"] != "", ( + f"{record['name']} now carries a second group's weakness — see " + "docs/smoke-agent-weaknesses.md before changing this record" + ) + assert isinstance(record["hours"], int), f"{record['name']} would break another group" + + +def test_g5_empty_field_does_not_touch_g1() -> None: + """G5's empty value must sit on `role`, which G1 never aggregates.""" + empty = [r for r in _records() if r["role"] == ""] + assert [r["name"] for r in empty] == ["Karl Jung"] + assert all(isinstance(r["hours"], int) for r in _records()), ( + "an empty `hours` would force a G1 fix to absorb G5's robustness" + ) + + +_EXPECTED_BASELINE: dict[tuple[str, str, str], float] = { + # Train shows two *kinds* of filter -- by department and by role -- so a + # general filter mechanism is the obvious fix. Validation holds new instances + # of those same two kinds, which a general fix reaches and a hardcoded one + # does not. This is what makes G1 a repair scenario rather than a + # generalization one; see docs/smoke-agent-weaknesses.md. + ("g1-aggregation", "train", "total-hours-research"): 0.0, + ("g1-aggregation", "train", "total-hours-engineers"): 0.0, + ("g1-aggregation", "train", "lookup-ada"): 1.0, + ("g1-aggregation", "validation", "total-hours-ops"): 0.0, + ("g1-aggregation", "validation", "total-hours-analysts"): 0.0, + ("g1-aggregation", "validation", "lookup-grace"): 1.0, + ("g2-name-patterns", "train", "lookup-obrien"): 0.0, + ("g2-name-patterns", "train", "lookup-zoe"): 0.0, + ("g2-name-patterns", "train", "lookup-ada"): 1.0, + ("g2-name-patterns", "validation", "lookup-ann-marie"): 0.0, + ("g2-name-patterns", "validation", "lookup-role-obrien"): 0.0, + ("g2-name-patterns", "validation", "lookup-grace"): 1.0, + ("g3-long-inputs", "train", "preamble-dept"): 0.0, + ("g3-long-inputs", "train", "preamble-role"): 0.0, + ("g3-long-inputs", "train", "plain-dept"): 1.0, + ("g3-long-inputs", "validation", "preamble-long-dept"): 0.0, + ("g3-long-inputs", "validation", "preamble-hours"): 0.0, + ("g3-long-inputs", "validation", "trailing-prose"): 1.0, + ("g4-dispatch-order", "train", "count-research"): 0.0, + ("g4-dispatch-order", "train", "count-ops"): 0.0, + ("g4-dispatch-order", "train", "lookup-ada"): 1.0, + ("g4-dispatch-order", "validation", "count-operators-ops"): 0.0, + ("g4-dispatch-order", "validation", "count-engineers-research"): 0.0, + ("g4-dispatch-order", "validation", "lookup-grace"): 1.0, + ("g5-edge-cases", "train", "missing-person"): 0.0, + ("g5-edge-cases", "train", "empty-role"): 0.0, + ("g5-edge-cases", "train", "lookup-ada"): 1.0, + ("g5-edge-cases", "validation", "missing-person-role"): 0.0, + ("g5-edge-cases", "validation", "missing-person-hours"): 0.0, + ("g5-edge-cases", "validation", "lookup-grace"): 1.0, +} + +GROUPS = ( + "g1-aggregation", + "g2-name-patterns", + "g3-long-inputs", + "g4-dispatch-order", + "g5-edge-cases", +) + + +@functools.cache +def _agent_class() -> Any: + """Import agent.py by path; it is not an installed package.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ.setdefault("TRACE_DIR", tempfile.mkdtemp(prefix="smoke-baseline-traces-")) + spec = importlib.util.spec_from_file_location("_smoke_baseline_agent", _EXAMPLE_DIR / "agent" / "agent.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module.ReportAgent + + +def _normalize(text: str) -> str: + """Mirror tests/test.sh: strip CR at end-of-line only, then trailing newlines. + + Deliberately not text.replace("\\r", "") -- that is `tr -d '\\r'`, which deletes + every carriage return and would let total=29 collapse into a passing + total=29. + """ + return re.sub(r"\r$", "", text, flags=re.MULTILINE).rstrip("\n") + + +def _reward_for(group: str, split: str, task_id: str) -> float: + """Replay the container verifier in-process for one task.""" + task = _EXAMPLE_DIR / "dataset" / "groups" / group / split / task_id + instruction = (task / "instruction.md").read_text(encoding="utf-8").strip() + expected = (task / "tests" / "expected.txt").read_text(encoding="utf-8") + written = _agent_class()().solve(instruction) + "\n" + return 1.0 if _normalize(written) == _normalize(expected) else 0.0 + + +@pytest.mark.parametrize(("key", "expected"), list(_EXPECTED_BASELINE.items())) +def test_baseline_rewards_are_pinned(key: tuple[str, str, str], expected: float) -> None: + """A failure here usually means someone "fixed" the agent. That is what it is for.""" + assert _reward_for(*key) == expected, ( + f"{key} no longer scores {expected} at baseline. The agent ships with deliberate " + "weaknesses; see plugins/nemo-experimentalist/docs/smoke-agent-weaknesses.md before " + "changing agent.py." + ) + + +@pytest.mark.parametrize("group", GROUPS) +@pytest.mark.parametrize("split", ["train", "validation"]) +def test_each_split_keeps_two_failures_and_one_control(group: str, split: str) -> None: + """The 2+1 shape is what distinguishes a real fix from a special case.""" + expected_ids = {task_id for (g, s, task_id) in _EXPECTED_BASELINE if g == group and s == split} + actual_ids = { + path.name + for path in (_EXAMPLE_DIR / "dataset" / "groups" / group / split).iterdir() + if (path / "task.toml").is_file() + } + assert actual_ids == expected_ids, ( + f"{group}/{split} drifted: missing={sorted(expected_ids - actual_ids)} " + f"unexpected={sorted(actual_ids - expected_ids)}" + ) + rewards = sorted(_reward_for(group, split, task_id) for task_id in expected_ids) + assert rewards == [0.0, 0.0, 1.0], f"{group}/{split} lost its two-failure/one-control shape: {rewards}" + + +@pytest.mark.parametrize( + ("answer", "expected_shape"), + [ + ("dept=research", 1.0), + ("count=3", 1.0), + ("names=Ada Lovelace", 1.0), + ("role=", 1.0), + ("I do not know how to answer that.", 0.0), + ], +) +def test_shape_metric_discriminates(answer: str, expected_shape: float) -> None: + """shape_ok must vary, or the second ranking dimension is dead weight. + + Mirrors the grep in tests/test.sh. A "did the agent write a file" metric was + rejected here: this agent always writes one, so it would be constant. + """ + actual = 1.0 if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", answer.splitlines()[0]) else 0.0 + assert actual == expected_shape diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py new file mode 100644 index 0000000000..7ff98b99e5 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py @@ -0,0 +1,311 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assert an Experimentalist run actually repaired the weakness it was given. + +Reward alone is not enough. A candidate can raise the number without fixing the +code, and a run can look healthy while measuring nothing, so each assertion below +targets a distinct way the result could be hollow. + +Every path and payload shape here is confirmed against a real run +(2026-08-05, G1 repair scenario, winner agent-1 at validation 1.000 from a +baseline of 0.333). Point SMOKE_EXPERIMENT_DIR at an experiment directory to run +these; they skip otherwise. + + SMOKE_EXPERIMENT_DIR=/tmp/smoke-g1-repair uv run pytest \\ + plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_gate.py -v +""" + +from __future__ import annotations + +import functools +import importlib.util +import json +import os +import re +import sys +import tempfile +from pathlib import Path +from typing import Any + +import pytest + +_EXPERIMENT_DIR = Path(os.environ.get("SMOKE_EXPERIMENT_DIR", "/nonexistent")) +_EO = "eval-and-optimize" + +# The repo's own copy of the fixture, used to replay held-out tasks against a +# candidate's code. The records file is the same one the run used: the task image +# is tagged by a content hash of it, so a drift would have failed an asset test. +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_RECORDS = _EXAMPLE_DIR / "dataset" / "_shared" / "records.json" +_G1_VALIDATION = _EXAMPLE_DIR / "dataset" / "groups" / "g1-aggregation" / "validation" + +# Which question this run answers. The two classes have *opposite* pass +# conditions, so a run is only meaningful once you say which one you started. +# repair -- train covers the general case; the winner must beat baseline +# generalization -- train covers only a narrow case; the baseline must be kept +_SCENARIO = os.environ.get("SMOKE_SCENARIO", "repair") + +# Assertion 2. A threshold, not equality: the agent under test is deterministic +# but the Experimentalist's own components are not. The observed delta was 0.667. +REWARD_DELTA_THRESHOLD = 0.3 + +# Assertion 4's matching rule. Two independent hits, so one incidental word does +# not satisfy it. The real analysis matched "total", "sum" and "aggregat". +ROOT_CAUSE_TERMS = ("total", "sum", "aggregat", "arithmetic") +MIN_ROOT_CAUSE_HITS = 2 + +# G1's validation split: two weakness tasks plus one control. Task names are the +# full `[task] name` from task.toml, not the directory name. +VALIDATION_WEAKNESS_TASKS = {"smoke/g1-total-hours-ops", "smoke/g1-total-hours-analysts"} +VALIDATION_CONTROL_TASKS = {"smoke/g1-lookup-grace"} + +# G4's split, used by the generalization assertions below. +G4_VALIDATION_CONTROL_TASKS = {"smoke/g4-lookup-grace"} + +pytestmark = pytest.mark.skipif( + not (_EXPERIMENT_DIR / _EO / "run.json").is_file(), + reason="no completed experiment at SMOKE_EXPERIMENT_DIR", +) + +repair_only = pytest.mark.skipif(_SCENARIO != "repair", reason="set SMOKE_SCENARIO=repair") + + +def _winner_label() -> str: + """Read the winning candidate's label from the run record.""" + run = json.loads((_EXPERIMENT_DIR / _EO / "run.json").read_text(encoding="utf-8")) + winner = run.get("winner_agent") + assert winner, f"run.json has no winner_agent: {sorted(run)}" + return str(winner) + + +def _agent_source(label: str) -> str: + return (_EXPERIMENT_DIR / _EO / "agents" / label / "agent.py").read_text(encoding="utf-8") + + +def _aggregate(label: str, dataset: str = "validation") -> dict[str, float]: + """Aggregate metrics for one evaluation. + + There is no top-level ``aggregate_metrics``. Harbor nests it under + ``stats.evals..metrics[0]``, and the eval key is Harbor's own, so + read the single entry rather than constructing the key. + """ + result = _EXPERIMENT_DIR / _EO / "results" / f"{label}-{dataset}" / "result.json" + evals = list(json.loads(result.read_text(encoding="utf-8"))["stats"]["evals"].values()) + assert len(evals) == 1, f"expected one eval in {result}, found {len(evals)}" + return evals[0]["metrics"][0] + + +def _per_task_rewards(label: str, dataset: str = "validation") -> dict[str, float]: + """Per-task reward, keyed by the full task name, from ``verifier_result.rewards``.""" + rewards: dict[str, float] = {} + for trial in sorted((_EXPERIMENT_DIR / _EO / "results" / f"{label}-{dataset}").glob("*/result.json")): + payload = json.loads(trial.read_text(encoding="utf-8")) + rewards[payload["task_name"]] = float(payload["verifier_result"]["rewards"]["reward"]) + return rewards + + +@functools.cache +def _agent_class(label: str) -> Any: + """Import a candidate's agent.py by path; it is not an installed package. + + `agent_source` points at the example's `agent/` directory and its *contents* + are copied to the candidate root, so agent.py sits directly under + `agents/