diff --git a/.copyrightignore b/.copyrightignore index 3ee8a7ba8b..6c3efb7050 100644 --- a/.copyrightignore +++ b/.copyrightignore @@ -3,6 +3,10 @@ web/vendor/ release/ .agents/skills/ e2e/ +plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json +plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md +plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/records.json +plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt packages/garak_api/garakapi/_config.py packages/garak_api/garakapi/_plugins.py packages/garak_api/garakapi/exception.py diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py index bb07311b05..a799618e6a 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py @@ -23,7 +23,6 @@ Task, TrialResult, ) -from nemo_experimentalist_plugin.experimentalist.components import cache from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( Diagnostic, @@ -486,7 +485,6 @@ async def _run( reporter.note(f"trace analysis failed for {ref}: {result}") analysis_statuses[task.id] = ("failed", str(result)) continue - cache.store(self.experiment_dir, cache.task_hash(f"eval_author:{ref}"), result) diagnostics.append((ref, result)) analysis_statuses[task.id] = ("completed", None) insight_suite.record_analysis(analysis_statuses) diff --git a/plugins/nemo-eval-author/tests/test_eval_author_agent.py b/plugins/nemo-eval-author/tests/test_eval_author_agent.py index 6fbff706e7..c4329c64c6 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_agent.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_agent.py @@ -262,7 +262,6 @@ def validate_metric_contracts( monkeypatch.setattr(eval_author_module, "TraceAnalyzer", FakeTraceAnalyzer) monkeypatch.setattr(eval_author_module, "validate_metric_contracts", validate_metric_contracts) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) monkeypatch.setattr(eval_author_module, "doc", lambda *_args, **_kwargs: "dataset docs") return calls diff --git a/plugins/nemo-eval-author/tests/test_plugin_boundary.py b/plugins/nemo-eval-author/tests/test_plugin_boundary.py index ce64320958..84713d6ecf 100644 --- a/plugins/nemo-eval-author/tests/test_plugin_boundary.py +++ b/plugins/nemo-eval-author/tests/test_plugin_boundary.py @@ -29,7 +29,6 @@ # adding one needs a deliberate argument for why duplicating the helper is worse. # # client -> make_client, the platform client factory -# ...components -> the cache module, for run artifacts # ...dataset_staging -> stage_eval_author_inputs # ...evaluator.base -> EvaluatorType # ...evaluator.factory -> DatasetFactory @@ -41,7 +40,6 @@ # ...reporting -> RunReporter (ASE-749: reuse Experimentalist narrator; do not duplicate) _BORROWED_BEHAVIOUR = { "nemo_experimentalist_plugin.client", - "nemo_experimentalist_plugin.experimentalist.components", "nemo_experimentalist_plugin.experimentalist.components.dataset_staging", "nemo_experimentalist_plugin.experimentalist.components.evaluator.base", "nemo_experimentalist_plugin.experimentalist.components.evaluator.factory", diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore b/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore new file mode 100644 index 0000000000..75f9fc9d3c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/.gitignore @@ -0,0 +1,7 @@ +# Generated by scripts/build_all_group.py, not authored. +# +# The loop takes a single --train-dataset path, so running several groups at once +# needs one combined directory. Its contents are byte-identical copies of the +# other groups, so committing it would double the dataset in the repo and put +# every group change in two places. Build it before running the full scenario. +dataset/groups/_all/ diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md b/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md new file mode 100644 index 0000000000..47dce91977 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md @@ -0,0 +1,75 @@ + + + +# smoke-agent + +## Prerequisites + +The task container ships Python, the standard library, and NOOA. It has no +network access and no API key, so anything that needs either fails outright. + +## Job + +Answer one question about the records file at `/app/data/records.json` and write +the single answer line to `/app/artifacts/output.txt`. + +## Interface + +- Invoked as `python main.py --prompt ""` with `/app` as the + working directory. +- Writes exactly one line, plus a trailing newline, to + `/app/artifacts/output.txt`. +- Writes an OTLP JSONL trace under `/app/traces/`. + +## Design + +`ReportAgent.solve` dispatches the instruction across an ordered list of +handlers and returns the first non-`None` answer, falling back to a fixed +string. Each handler matches the question with a regular expression, looks the +answer up in the records, and formats one line as `=`. + +The records are a list of objects with `name`, `dept`, `role`, and `hours`. +`FIELD_ALIASES` maps the word a question uses to the key the records store it +under, so the answer line is always keyed by the canonical field name. + +## Missing and empty values + +A question may name a person the records do not contain, or ask for a field +whose stored value is an empty string. Both are answered the same way: the value +is the word `unknown`, so the line reads `dept=unknown`. This is part of the +output contract and is compared byte-for-byte like any other answer — the +sentinel is `unknown` exactly, not `n/a`, `none`, or the empty string. + +## Answer keys + +The key on the left of the `=` names what the answer *is*, not the field it came +from. The vocabulary is fixed: + +- a value read from one record uses that field's own name — `dept=`, `role=`, + `hours=` +- a sum over records is reported as **`total=`**, whatever field was summed and + however the records were selected +- a number of records is reported as `count=` + +Keys are compared byte-for-byte like the rest of the line, so `hours=99` is wrong +where `total=99` is expected, even though the number is right. + +## Constraints — these are hard requirements + +- **The agent is deterministic and offline.** The same instruction must always + produce the same answer. Reward differences between candidates must come from + code changes, never from sampling. +- **No LLM.** Do not add a `@strategy` method, an LLM-backed handler, a subagent + with its own model, or a model swap. The task container has no network and no + API key, so such a change fails outright — but more importantly, being + reproducible is this agent's entire contract. +- Standard library plus NOOA only. No new dependencies. +- Do not edit `/app/data/records.json`. It is task-supplied input, not agent + code, and it is not part of this directory. +- The output line is compared byte-for-byte against the task's expected value, + so trailing whitespace, extra lines, and changes to the `=` form + all count as wrong answers. + +## Next steps + +Any implementation change must still satisfy every constraint above. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/README.md b/plugins/nemo-experimentalist/examples/smoke-agent/README.md new file mode 100644 index 0000000000..4b93b00fa7 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/README.md @@ -0,0 +1,374 @@ + + + +# smoke-agent + +A fast fixture for exercising the Experimentalist loop end to end. It exists to +make a full round cheap enough to run while refactoring, and to let a run be +checked for more than "it completed". + +The agent is **wrong on purpose**. Five known weaknesses, each paired with a +group of Harbor tasks that surfaces it, so a run can be asserted to have +*repaired* something rather than merely completed. + +> **Nothing inside `agent/` or `AGENT-SPEC.md` may describe what this fixture +> measures.** `agent_source` points at `agent/`, so it is copied into every +> candidate workspace and read by the Coder, and the spec reaches the LLM +> components by a separate route. A description in either would hand the Coder +> the diagnosis the fixture exists to test. Everything else here — this file, +> `configs/`, `scripts/`, `dataset/` — is never copied and can say whatever is +> useful, which is why the weakness detail below lives in this file. +> +> **Do not repair the five weaknesses in the agent.** A well-meaning cleanup +> silently destroys what the fixture measures: with the weakness gone, the +> baseline passes, the Analyst gets no failing trace, and a run that does nothing +> looks identical to a run that works. + +## Prerequisites + +- A local NeMo Platform with default and fast Model Entities selected by `nemo setup` +- Docker Engine and Docker Sandboxes (`sbx`) +- `uv` + +## Design + +- **The agent makes no model calls.** Handlers are regular expressions plus a + dict lookup over `dataset/_shared/records.json`, so the same instruction always + produces the same answer and the only stochastic component in a run is the + Experimentalist itself. The `CompletionClient` in `agent.py` points at an + unroutable address: an accidental model call fails loudly rather than quietly + making the agent nondeterministic. +- **Task definitions are local and checked in.** No registry, no NeMo Platform + for Mode 2, no network inside the task container. +- **One prebuilt image serves every task**, referenced by + `[environment].docker_image` rather than a per-task Dockerfile. Its tag is a + content hash of the Dockerfile and the records file, so forgetting to rebuild + fails a test instead of silently running against stale data. + +## Layout + +```text +agent/ ONLY this is copied to the Coder (agent_source) +agent/agent.py the code under optimization +agent/main.py container entry point +agent/harbor_wrapper.py Harbor upload + exec adapter +AGENT-SPEC.md behaviour contract read by the LLM components +optimizer.yaml profile: agent source, spec, g1 datasets +optimizer-full.yaml profile: the generated combined datasets +optimizer-generalization.yaml profile: same agent, g4 datasets (see Scenarios) +configs/short.yaml loop settings shared by the per-group gate checks +configs/full.yaml loop settings for the multi-round scenario +dataset/_shared/ canonical Dockerfile, records, verifier +dataset/tasks.json authored task values used to render every curated task +dataset/task-template/ one Harbor task shape, also used by insight mode +dataset/groups/ GENERATED Harbor task sets, gitignored +dataset/insights/ insight mode only, frozen analyst output +scripts/render_tasks.py render every curated task from the template +scripts/build_image.py build the image and render/stamp every task +scripts/build_all_group.py assemble the combined group the full scenario runs +scripts/record_traces.py evaluate a group and ingest its traces +``` + +## Running it + +Run the loop **inside a Docker sandbox**. Clone mode gives it a private writable +clone rather than write access to the host checkout, which keeps a run from +touching this directory. + +```bash +repo="$(git rev-parse --show-toplevel)" + +sbx create --clone --name nemo-experimentalist shell "$repo" +``` + +> **In a git worktree?** `--clone` refuses to run there, so bind-mount instead +> with `sbx create --profile developer --name nemo-experimentalist shell "$(pwd)"`. +> That gives the sandbox write access to this checkout, so the source-mutation +> protection clone mode provides is gone. Check `git status` after a run. + +```bash +# The image lives in the sandbox's own Docker daemon, so build it there. +sbx exec --workdir "$repo" nemo-experimentalist bash -lc \ + 'cd plugins/nemo-experimentalist/examples/smoke-agent && uv run --no-project scripts/build_image.py' +``` + +`build_image.py` first renders `dataset/groups/` from `dataset/tasks.json` and +`dataset/task-template/`. Those directories are gitignored build output: edits +there are discarded on the next build, so change the manifest or the template +instead. + +Then run a scenario. `--with ./plugins/nemo-agents` is required: the `agents` +command group lives in a separate workspace package, and without it the CLI fails +with `No module named 'nemo_agents_plugin'`. + +```bash +sbx exec --workdir "$repo" \ + --env UV_PROJECT_ENVIRONMENT=/home/agent/.venvs/nemo-platform \ + nemo-experimentalist \ + bash -lc 'uv run --frozen --python 3.13 \ + --package nemo-experimentalist-plugin --with ./plugins/nemo-agents \ + nemo agents experimentalist run \ + --profile plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml \ + --no-insight \ + --base-url http://host.docker.internal:8080 \ + --config plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml \ + --experiment-dir /tmp/smoke-repair' +``` + +Copy the experiment directory back out with `sbx cp` to check the result. + +**Models come from the platform, not from this example.** The Experimentalist +reads a *Model Entity* pair from the active CLI context — `default_model` and +`fast_model`, the second falling back to the first when unset — so `nemo setup` +is what configures them. The value is an entity id of the form +`/`, not a Litellm routing string; list what your Platform has +with `nemo models list --all-pages`. Preflight only checks that a value is *set*, +so a typo passes `doctor` and fails at the first LLM call, minutes into a run. + +**`--base-url` points at a platform, and the sandbox is not the host.** A +container's `localhost` is itself, so a platform on your machine is reached at +`host.docker.internal:8080`. Without a reachable platform the run still works, +but every projection onto native `ExperimentGroup`/`Experiment` entities fails +and logs `[MIRROR] projection failed`. That is best-effort and never fails a run, +so a log full of it means the platform was unreachable, not that anything went +wrong. + +### Run the loop tests + +The loop tests are developer-invoked and execute model-written shell inside the +named sandbox. Create the `nemo-experimentalist` sandbox with the command above +first, then: + +```bash +SANDBOX_VM_ID=nemo-experimentalist uv run --frozen pytest \ + plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_1_loop_e2e.py \ + plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_2_loop_e2e.py \ + -m e2e -n 4 --dist loadgroup +``` + +Pytest writes logs and downloaded artifacts under its temporary test directory. +It does not retry failed runs. + +## Scenarios + +**The profile picks the scenario, not the config.** What separates a repair run +from a generalization one is the split it runs against, so each has its own +profile: + +| Profile | Config | Rounds | A healthy run ends with | +| --- | --- | --- | --- | +| `optimizer.yaml` | `short.yaml` | 2 | the winner beating the baseline | +| `optimizer-generalization.yaml` | `short.yaml` | 2 | the baseline correctly retained | +| `optimizer-full.yaml` | `full.yaml` | up to 5 | every task in the combined group passing | + +The first two are opposite tests, so a run is only meaningful once you know which +one you started — and the config cannot tell you, because both use the same one. +That is not an oversight: the scenario config carries loop settings only, and the +schema has no dataset field at all. + +`full.yaml` is the only one that exercises the evolutionary machinery — survivors +carried between rounds, ranking over more than two candidates, and the +convergence check. It runs against `dataset/groups/_all`, which is **generated +and gitignored**; build it first, or the run loads zero tasks and reports +`No tasks matched the filter(s)` rather than erroring: + +```bash +sbx exec --workdir "$repo" nemo-experimentalist bash -lc \ + 'cd plugins/nemo-experimentalist/examples/smoke-agent && uv run --no-project scripts/build_all_group.py' +``` + +Rerun that after changing any group. + +## Groups + +Every group is a self-contained train/validation pair of six tasks — per split, +two that fail at baseline and one that already passes. That control is the point: +it makes a destructive fix cost reward instead of passing unnoticed, and it means +a group's score can fall as well as rise. Every group except `g4-dispatch-order` +also carries an `insight-evidence` split of five tasks, which is what an +Insight-driven (mode 1) run and the Analyst read. + +| Group | What a run against it tests | Backs | In `_all` | +| --- | --- | --- | --- | +| `g1-aggregation` | **Repair.** A capability that is absent rather than wrong. | `short.yaml` | yes | +| `g2-name-patterns` | Widening a pattern that is too narrow. | `full.yaml` | yes | +| `g3-long-inputs` | A constant rather than logic — the one `edit_config` in the set, so a run exercises a different path through the Coder. | `full.yaml` | yes | +| `g4-dispatch-order` | **Generalization.** The tempting fix passes train and fails validation, so a healthy run *keeps the baseline*. | `short.yaml` | no | +| `g5-edge-cases` | Several changes that score only when all are made, whose partial states are indistinguishable in the output. The hardest here. | `short.yaml` | no | + +The groups are not interchangeable. Several independently addressable groups let +a single round produce genuinely *different* candidates, and let a later round +inherit one fix and add another; they differ in the kind of edit they call for, +so a run exercises more than one path through the Coder; and they differ in +difficulty, so an early round has something to find while the harder ones keep +the ceiling out of reach of a shallow fix. + +Two are held out of the combined set, for different reasons. +`g4-dispatch-order`'s healthy outcome — baseline retained — is the opposite of +the combined scenario's, and one run cannot assert both. `g5-edge-cases` is only +reachable with trajectory scoring on, and the combined scenario runs with it off: +measured over runs made after the spec stated the sentinel, no candidate closed +it without a goal tree and most did with one. Trajectory scoring is not +dependable enough to leave on yet, so the group is out until it is. +`build_all_group.py` records the numbers and the condition for putting it back. +Run it on its own with `short.yaml` in the meantime — it is a repair-shaped +split. + +## The five weaknesses + +Tasks are authored in `dataset/tasks.json`; the agent is `agent/agent.py`. + +### G1 — no aggregation capability + +**What.** Nothing sums or averages a numeric field. `solve` dispatches over +`handle_lookup`, `handle_list`, `handle_count`; none matches "what is the total +`` in the …", so every aggregation question falls through to `FALLBACK`. + +**Odd one out.** This is the only *missing* capability; G2–G5 are flawed code +paths that exist. There is no wrong code to find, only absent code, so G1's tasks +exercise the Proposer more than the Coder. + +**Tasks.** An aggregation question scopes its sum either by department or by +role, and **both splits carry both scopes**. Train sums by department +(`total=29`, research) and by role (`total=20`, engineer); validation sums by +role (`total=9`, analyst) and by department (`total=13`, ops). A fix that +hardcodes either scope therefore fails inside its own split rather than surviving +to validation: the pressure to read the scope from the question is applied +immediately, not held out. + +**A correct repair** adds a handler that sums a field over a record subset, with +the scope taken from the question rather than assumed, and inserts it into the +dispatch tuple. + +### G2 — name pattern too narrow + +**What.** `LOOKUP_RE`'s name group is `([A-Za-z ]+)`, so any name carrying an +apostrophe, a hyphen, or a non-ASCII character fails to match at all and falls +through to `FALLBACK`. + +**Tasks.** `O'Brien`, `Zoë Washington`, `Ann-Marie Cruz`. Controls are +plain-ASCII lookups that already work. + +**A correct repair** widens the character class. Note `str.isalpha()` is *not* a +valid test for "would this name match" — it is Unicode-aware and accepts `Zoë` +while the agent's ASCII class rejects it. The guard test uses the agent's own +class for exactly this reason. + +### G3 — instruction clipped before dispatch + +**What.** `solve` truncates to `MAX_INSTRUCTION_CHARS = 240` before dispatching, +so a question preceded by a long preamble is cut off and matches nothing. + +**Tasks.** A ~320-character reporting-policy preamble in front of a question that +works without it. `preamble-long-dept` doubles the preamble to roughly 650 +characters, so a repair that only nudges the limit upward still fails it. + +**The control is load-bearing.** `trailing-prose` puts the question *first* and +the prose after, so it passes at baseline **and** would break under a fix that +reads only the tail of a clipped instruction. It catches a specific bad repair. + +**A correct repair** raises or removes the limit. + +### G4 — dispatch order shadows the count handler + +**What.** `LIST_RE` is `(?:list|how many) .*? in the (\w+) department` — the +`how many` alternative belongs to `COUNT_RE` — and `solve` consults +`handle_list` first. Counting questions are therefore answered with a list of +names. + +**Odd one out.** This is the only group whose failure is a *wrong-shaped answer* +rather than the fallback, so it scores `reward 0` with `shape_ok 1.0`. G1, G2, +G3 and G5's missing-record mode all fall back to prose and score `shape_ok 0`. +Measured: G4 train aggregates to `reward 0.333, shape_ok 1.0`, G5 validation to +`0.333, 0.333`. The two groups are separable from aggregates alone. + +**Two halves, one repair.** `LIST_RE` and the dispatch tuple order are a matched +pair. Changing either alone leaves the shadowing in place. + +**Tasks.** Train counts people per department; validation counts *by role +within* a department, which matches `LIST_RE` but **not** `COUNT_RE`. So +reordering the tuple alone still fails validation; the count pattern has to widen +too. + +### G5 — missing and empty data not handled + +**What.** Two distinct failure modes: + +- `handle_lookup` resolves a record with `next(r for r in … if r["name"] == name)`, + which **raises** when the name is absent. `solve`'s top-level `except` catches + it and returns `FALLBACK`. +- An empty stored value yields an empty right-hand side (`role=`) rather than a + documented `role=unknown`. + +**Why the catch exists.** Without it the process exits non-zero, the wrapper +raises, and Harbor records a *harness error* rather than a scored 0 — which +leaves the Analyst nothing to read. Verified: G5 trials complete with +`status=completed` and reward 0. The catch controls the exit code, not the trace; +NOOA already records the exception on the failing handler's span. + +**Tasks.** Names absent from the records, and Karl Jung's empty `role`. Expected +answers use `unknown`. + +## Orthogonality, including the data + +Each group's tasks must supply evidence for **its own weakness only**, so a run's +failing traces point at one root cause. That extends to +`dataset/_shared/records.json`, which all five groups share — the data is a +coupling surface as much as the code is. + +Current assignment, pinned by `test_smoke_agent_baseline.py`: + +| Record | Serves | Must stay | +| --- | --- | --- | +| Ada Lovelace, Grace Hopper | controls, G1 engineer sum | plain ASCII, non-empty, int hours | +| Zoë Washington, O'Brien, Ann-Marie Cruz | G2 | non-empty `role`, int `hours` | +| Karl Jung | G5 empty-field mode | empty `role`, **int `hours`** | + +Karl Jung's `hours` is `0`, not empty, and that is deliberate: an empty `hours` +sits in `ops`, which G1 aggregates, so it would force a G1 fix to absorb G5's +robustness. + +This has already gone wrong once through a *task* rather than the data. G3's +`preamble-long-dept` originally looked up a name absent from the records, so +closing G3 alone would have left it failing on G5's missing-record path. Check +both when adding either. + +## Checking a run + +The Mode 1 and Mode 2 E2E tests check the experiment artifacts themselves. They +verify source changed, held-out tasks pass, controls do not regress, and the +Analyst named each group's problem. They also verify that G4 keeps the baseline +when a train-only change fails to generalize. + +**Also run the guard suite after every loop run:** + +```bash +uv run pytest plugins/nemo-experimentalist/tests/experimentalist/ -k smoke -q +``` + +`test_smoke_agent_baseline.py` and `test_smoke_agent.py` pin the agent behavior +and the records table above. A failure almost always means the fixture itself +changed, which makes every later run meaningless while still looking healthy. Do +not "fix" the agent to make it pass; confirm against the weakness descriptions +above first. + +`test_smoke_agent_assets.py` pins the other half — the image tag, the task +template, and the tree rendered from the manifest — so run it after changing any +of those, not only after a loop. + +What is **not** pinned: no test asserts which task sits in which split. The split +assignments described above can drift without a test failing, so check them +against `dataset/tasks.json` whenever you edit the manifest. + +## Timings + +Measured on 2026-08-05, one round, two candidates, three tasks per split: + +| | | +| --- | --- | +| One split evaluated in containers | ~15 s | +| A full round end to end | ~18 min | + +Container evaluation is negligible by design. The cost is the Experimentalist's +own components — the architecture doc, Analyst, Proposer, and one Coder pass per +candidate — which is the part under test and cannot be optimized away here. \ No newline at end of file diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py new file mode 100644 index 0000000000..e1d300a1c8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/agent.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Answer one question about a records file. + +Deterministic by construction: no LLM call, no network, standard library plus +NOOA only. The same instruction always produces the same answer. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path + +from nooa import Agent +from nooa.tracing import enable_tracing, exporters +from nooa.unifiedllm import CompletionClient + +enable_tracing(exporters=[exporters.jsonl(trace_dir=os.environ.get("TRACE_DIR", "/app/traces/"))]) + +logger = logging.getLogger(__name__) + +RECORDS_PATH = Path(os.environ.get("RECORDS_PATH", "/app/data/records.json")) +FALLBACK = "I do not know how to answer that." + +# Bound how much of the instruction we scan, so an oversized input cannot make +# the regex pass expensive. +MAX_INSTRUCTION_CHARS = 240 + +# Maps the word a question uses to the key the records store it under, so the +# answer line is always keyed by the canonical field name. +FIELD_ALIASES = { + "department": "dept", + "dept": "dept", + "role": "role", + "hours": "hours", +} + +# The question forms this agent recognizes. +LOOKUP_RE = re.compile(r"what is the (\w+) of ([A-Za-z ]+)\?", re.IGNORECASE) +LIST_RE = re.compile(r"(?:list|how many) .*? in the (\w+) department", re.IGNORECASE) +COUNT_RE = re.compile(r"how many people are in the (\w+) department", re.IGNORECASE) + +# Never called. The address is unroutable so an accidental model call fails +# loudly rather than silently making the agent nondeterministic. +_DUMMY_LLM = CompletionClient(model="none", api_key="unused", api_base="http://127.0.0.1:1/v1") + + +class ReportAgent(Agent, llm=_DUMMY_LLM): + """Answer one question about the records file.""" + + _enable_tracing = True + + def __init__(self, **kwargs: object) -> None: + """Load the records the task environment supplied.""" + super().__init__(**kwargs) + self._records: list[dict] = json.loads(RECORDS_PATH.read_text(encoding="utf-8")) + + def solve(self, instruction: str) -> str: + """Return the single answer line this instruction asks for.""" + instruction = instruction[:MAX_INSTRUCTION_CHARS] + try: + for handler in (self.handle_lookup, self.handle_list, self.handle_count): + answer = handler(instruction) + if answer is not None: + return answer + except Exception: # noqa: BLE001 + # A handler fault must not take the process down: the caller still + # needs an answer line written, and a non-zero exit would be reported + # as a harness error rather than a scored result. + logger.exception("handler failed") + return FALLBACK + + def handle_lookup(self, instruction: str) -> str | None: + """Return `=` for the named record, or None if not a lookup.""" + match = LOOKUP_RE.search(instruction) + if match is None: + return None + field = FIELD_ALIASES.get(match.group(1).lower()) + if field is None: + return None + name = match.group(2).strip() + record = next(r for r in self._records if r["name"] == name) + return f"{field}={record[field]}" + + def handle_list(self, instruction: str) -> str | None: + """Return `names=` for a department, or None.""" + match = LIST_RE.search(instruction) + if match is None: + return None + dept = match.group(1).lower() + return "names=" + ",".join(r["name"] for r in self._records if r["dept"] == dept) + + def handle_count(self, instruction: str) -> str | None: + """Return `count=` for a department, or None.""" + match = COUNT_RE.search(instruction) + if match is None: + return None + dept = match.group(1).lower() + return f"count={sum(1 for r in self._records if r['dept'] == dept)}" diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py new file mode 100644 index 0000000000..38d6f54055 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/harbor_wrapper.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor adapter for the smoke agent. + +No dependency install: NOOA is already in the task image. Harbor collects +/app/artifacts (declared by each task.toml) and /app/traces (injected by the +evaluator), so nothing is copied by hand. +""" + +from __future__ import annotations + +import fnmatch +import logging +import shlex +from pathlib import Path + +from harbor import AgentContext, BaseAgent, BaseEnvironment + +logger = logging.getLogger(__name__) + +AGENT_DIR = Path(__file__).parent + +# Never uploaded into the task container: optimizer bookkeeping, caches, and +# anything holding credentials. +# +# Most of these can no longer be siblings -- this file lives in the agent +# directory that `agent_source` points at, which holds the agent and nothing +# else. They are kept because the directory is a candidate workspace at run time +# and can accumulate caches, and because a credential file inside an agent +# directory must not travel regardless of layout. +EXCLUDE = { + "eval-and-optimize", + "__pycache__", + ".git", + ".claude", + ".uv", + ".venv", + ".env", + ".env.example", + "traces", + "artifacts", + "dataset", + "scripts", +} +EXCLUDE_GLOB = {"output.*", "*.md"} + + +class SymlinkedUploadError(RuntimeError): + """A selected upload path is, or contains, a symlink.""" + + +def _reject_symlinks(entries: list[Path]) -> None: + """Raise if any selected entry is a symlink or holds one at any depth. + + Args: + entries: Top-level paths selected for upload. + + Raises: + SymlinkedUploadError: naming the first offending path. + """ + for entry in entries: + offenders = [entry] if entry.is_symlink() else [] + if entry.is_dir() and not entry.is_symlink(): + offenders.extend(child for child in entry.rglob("*") if child.is_symlink()) + if offenders: + raise SymlinkedUploadError( + f"refusing to upload {offenders[0]}: it is a symlink, and following it would copy " + "host files outside the agent directory into the task container" + ) + + +class WrappedAgent(BaseAgent): + """Upload this agent directory into the container and run one task.""" + + @staticmethod + def name() -> str: + """Return the agent name Harbor records for a trial.""" + return "smoke-agent" + + def version(self) -> str | None: + """Return the agent version Harbor records for a trial.""" + return "1.0.0" + + async def setup(self, environment: BaseEnvironment) -> None: + """Upload the agent's source files. NOOA is already installed in the image.""" + selected = [ + entry + for entry in AGENT_DIR.iterdir() + if entry.name not in EXCLUDE and not any(fnmatch.fnmatch(entry.name, pattern) for pattern in EXCLUDE_GLOB) + ] + # Refuse symlinks before uploading anything. upload_dir follows them, so a + # link anywhere in a selected subtree would copy host files into the + # container. Scanned up front so a rejection cannot leave a half-populated + # /app. + _reject_symlinks(selected) + for entry in selected: + if entry.is_file(): + await environment.upload_file(entry, f"/app/{entry.name}") + elif entry.is_dir(): + await environment.upload_dir(entry, f"/app/{entry.name}") + logger.info("[setup] uploaded agent sources to /app") + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Execute the agent on *instruction* inside the task container.""" + session_id = self.session_id or "local" + proc = await environment.exec( + f"cd /app && python main.py --prompt {shlex.quote(instruction.strip())} " + f"--session-id {shlex.quote(session_id)}" + ) + + # Nothing is copied here: Harbor collects /app/artifacts and /app/traces + # after this method returns, per the declarations described above. + context.metadata = { + "stdout": proc.stdout, + "stderr": proc.stderr, + "returncode": proc.return_code, + } + if proc.return_code != 0: + raise RuntimeError(f"Agent process failed with exit code {proc.return_code}: {proc.stderr or proc.stdout}") diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py b/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py new file mode 100644 index 0000000000..2c98fcba9a --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/agent/main.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Container entry point: solve one task and write the answer line.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from agent import ReportAgent + +ARTIFACTS_DIR = Path(os.environ.get("ARTIFACTS_DIR", "/app/artifacts")) +OUTPUT_PATH = ARTIFACTS_DIR / "output.txt" + + +def main() -> None: + """Run the agent on --prompt and write /app/artifacts/output.txt.""" + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", required=True) + parser.add_argument("--session-id", default=os.environ.get("HARBOR_SESSION_ID", "local")) + args = parser.parse_args() + + answer = ReportAgent().solve(args.prompt) + + ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH.write_text(answer + "\n", encoding="utf-8") + print(f"answer={answer!r} output={OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml new file mode 100644 index 0000000000..dbc775e3a6 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Full scenario: several groups at once, multiple rounds, terminator decides when +# to stop. Point it at dataset/groups/_all. +# +# Two groups are held out of it -- one whose pass condition is the opposite of +# this one's, and one that is only reachable with trajectory scoring on, which it +# is not here. build_all_group.py has the evidence and the conditions for putting +# the second one back. Every task the combined set does contain is meant to be +# reachable, so a run that leaves one failing is a finding rather than an +# expected shortfall. +# +# What only this scenario exercises: more than one survivor, ranking over more +# than two candidates, and the convergence check deciding when to stop. short.yaml +# now runs two rounds, so carrying a survivor forward is no longer the difference +# between them -- depth is. It keeps a single survivor, a fixed two rounds, and no +# terminator, so none of the machinery above is reached. +# +# EXPECT HOURS, NOT MINUTES. Roughly 25 min per round at three candidates: a bit +# over an hour if the terminator stops at three rounds, around two if it reaches +# the ceiling. That cost buys +# something tau-style benchmarks cannot: the expected outcome is known up front, +# so the result is checkable rather than merely plausible. + +# A ceiling, not a target. The terminator is expected to stop first; reaching the +# ceiling is itself a finding. +max_rounds: 5 + +# Default. The convergence check may not stop the run before this many rounds. +min_rounds_before_stopping: 3 + +max_survivors: 3 + +# Three per round. Enough for the Proposer to try genuinely different edits in a +# round -- past runs landed three separate improvements in one -- without paying for +# candidates that mostly duplicate each other. The Proposer may return fewer, and +# often does in later rounds; only an empty proposal is fatal. +max_candidates: 3 + +# Both metrics have a known ceiling in this deterministic fixture. This lets the +# loop stop once a surviving candidate passes every task and keeps the required +# output shape. `full.yaml` does not inherit `short.yaml`, so it declares the +# same contract explicitly. +objective_function: + - name: reward + direction: maximize + target: 1.0 + - name: shape_ok + direction: maximize + target: 1.0 + +# Whole split. Six failing tasks and three that already pass, so there is a +# gradient to climb and a cost to breaking something that worked. +max_train_batch_tasks: null + +# Trajectory scoring is OFF, and that also switches off the goal tree it feeds. +# +# Not because it cannot work here. This comment used to say the scorer was +# defeated by the agent's determinism -- candidates indistinguishable, nothing to +# order by -- and that was wrong. The traces separate them plainly: the call +# graph names every method that ran and its status, so a candidate whose lookup +# errors is obvious beside one whose lookup returns a value. In a verification +# run the scorer ranked two candidates that were tied at 1.000 on reward, citing +# exactly that. +# +# What actually failed was the citation rule. Every score had to be grounded in +# span IDs, and the only sanctioned lookups are indexed by turn -- which an agent +# that makes no LLM calls never produces. The scorer could not satisfy its own +# contract, retried to the CodeAct ceiling, and raised GenerationError, which +# ended the run. Fixed in trace_scorer.py. +# +# It stays off because it is not yet dependable, and both failure modes are +# silent: a goal tree rejected on node count disables it for an entire run +# without saying so, and in the run that verified the fix it scored three +# candidates and returned nothing for the fourth, which then won. +# +# Consequence: trajectory scoring and the goal tree are NOT covered by this +# scenario. Multiple rounds, survivors, ranking, and the convergence check are. +# Turning it on is worthwhile once it is dependable -- build_all_group.py holds +# g5 out of the combined set for the same reason. +disable_trajectory_scoring: true + +# The one that matters here: the terminator deciding when to stop is the point. +disable_convergence_check: false + +evaluator: + n_attempts: 1 + n_concurrent_trials: 5 + quiet: true diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml new file mode 100644 index 0000000000..5a9318b2b8 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Two rounds, two candidates, whole train split. The per-group gate checks share +# this file, because the loop settings they want are identical. What separates +# them is the profile: +# +# optimizer.yaml -> g1-aggregation, the winner must beat baseline +# optimizer-generalization.yaml -> g4-dispatch-order, the baseline must be kept +# +# There were once two files here, differing in `max_train_batch_tasks`: the +# generalization scenario sampled a single train task on the theory that seeing +# one example made an over-narrow fix likely. That theory was wrong. What decides +# the scenario is how the splits are built -- whether train shows enough for a +# general fix to be reachable -- and sampling only dropped tasks, once excluding +# the control. Setting it back to `null` left two identical files, and a README +# claiming they differed. Merged rather than left as a trap. + +# Two, not one. A single round scores whatever the Coder guessed first, and a +# near-miss is indistinguishable from no attempt: a g3 run produced a candidate +# that split the instruction and kept both ends, which is the right idea reached +# the wrong way, and the round ended there. A second round breeds from that +# survivor with the first round's analysis in hand, so a partial fix gets a chance +# to be finished rather than being recorded as a failure. +# +# It also makes the gate less noisy. At two candidates a single round lands the g1 +# repair about 80% of the time, so a red result was as likely to be variance as a +# regression -- which is useless for a check whose whole job is to be believed. +max_rounds: 2 + +# Default, and inert while the convergence check is off below. It is not inert if +# that is ever turned back on: at 1, `cutoff_round` is the latest round, so `old` +# holds every scored node, the two Pareto fronts are the same set, and the check +# returns converged after round 1 whatever the scores. That would make +# `max_rounds: 2` dead config. Anyone enabling the check here must raise this to 2 +# in the same edit. +min_rounds_before_stopping: 1 + +max_survivors: 1 + +# Two candidates so ranking and survivor selection do real work. With one, +# nothing is compared against anything and the two metric keys buy nothing. +max_candidates: 2 + +# Whole train split, for both scenarios. The Coder must see every kind of case +# the split carries, and the control must be present so a destructive fix costs +# reward. A number here silently drops tasks -- 2 excluded the control in a real +# run, leaving the round with nothing failing to analyze. +max_train_batch_tasks: null + +# Both metrics have a known ceiling in this deterministic fixture. Stop before +# another round once one eligible candidate passes every task without changing +# the required output shape. +objective_function: + - name: reward + direction: maximize + target: 1.0 + - name: shape_ok + direction: maximize + target: 1.0 + +disable_trajectory_scoring: true + +# Off, and it should stay off at this depth. The terminator only tests for +# *stagnation* -- whether any new candidate reached the Pareto front, then an LLM +# judgement on whether the round analysis reads as a plateau. It has no notion of +# the objective being satisfied, so a round that goes 0.333 -> 1.000 does not stop +# early; it costs an extra model call and runs round 2 anyway. +# +# "Qualitatively plateaued" reads like it might cover a maxed-out score, but the +# terminator skill it is judged against lists only stagnation signals -- a repeated +# root cause, survivors converging on the same mutation, diagnostics thinning -- and +# names "describes ongoing improvement" as a reason to *continue*. A run that has +# just solved everything does stop eventually, one round later, when nothing new +# reaches the front: as stagnation, not as success. +# +# The case it would stop on is the opposite one: a round-1 candidate that ties the +# baseline reads as a plateau, and that is precisely the near-miss round 2 exists +# to finish. Enabling it here would cut the runs that need the second round and +# leave the ones that do not. +# +# `full.yaml` is where the terminator earns its place: five rounds, three +# candidates, and stopping when the front genuinely stops moving. +disable_convergence_check: true + +evaluator: + n_attempts: 1 + n_concurrent_trials: 3 + quiet: true + +# Insight mode materializes and deeply examines these representative failures +# before the optimization loop begins. Five matches the five distinct failure +# shapes recorded for every smoke-agent group. +eval_author: + max_traces: 5 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile new file mode 100644 index 0000000000..42085c9251 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/Dockerfile @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The whole task environment, built once and referenced by every task through +# [environment].docker_image. NOOA is baked in rather than installed during +# Harbor setup, so per-trial setup stays near zero and only a cached image build +# pays. The pinned revision must match the workspace root pyproject.toml; a test +# asserts it. + +FROM python:3.12-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv + +# `uv pip install --system` rather than `uv sync`: there is no project here, just +# one pinned dependency going into the image's own interpreter. +RUN uv pip install --system --no-cache \ + "nooa[tracing] @ git+https://github.com/NVIDIA-NeMo/labs-OO-Agents.git@6e0274dd03f883254a084cfb9f871ea580e03434" + +WORKDIR /app +RUN groupadd --gid 10001 smoke-agent \ + && useradd --uid 10001 --gid smoke-agent --create-home --shell /usr/sbin/nologin smoke-agent \ + && mkdir -p /app/artifacts /app/traces /app/data \ + && chown -R smoke-agent:smoke-agent /app +COPY --chown=smoke-agent:smoke-agent records.json /app/data/records.json +USER smoke-agent diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json new file mode 100644 index 0000000000..10f5cad8e7 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json @@ -0,0 +1,8 @@ +[ + {"name": "Ada Lovelace", "dept": "research", "role": "engineer", "hours": 12}, + {"name": "Grace Hopper", "dept": "research", "role": "engineer", "hours": 8}, + {"name": "Zoë Washington", "dept": "research", "role": "analyst", "hours": 9}, + {"name": "O'Brien", "dept": "ops", "role": "operator", "hours": 5}, + {"name": "Ann-Marie Cruz", "dept": "ops", "role": "operator", "hours": 8}, + {"name": "Karl Jung", "dept": "ops", "role": "", "hours": 0} +] diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh new file mode 100755 index 0000000000..f17fa04dad --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -L "$OUTPUT" ] || [ -L /app/artifacts ]; then + # The agent owns /app/artifacts and this runs afterwards, so it could replace the + # answer with a symlink at the expected fixture: `-f`, `head`, `sed` and `cmp` all + # follow links, so the answer key would be compared against itself and score 1.0 + # without anything being solved. `-L` is the one test that does not follow, and it + # has to come first for the same reason. The directory is checked too, since + # linking it moves the whole path somewhere else. + # + # This catches symlinks, not hard links -- one of those is indistinguishable from a + # regular file here. It needs both paths on one filesystem, which separate mounts + # prevent, so it is left as a known limit rather than guessed at. + echo "FAIL: ${OUTPUT} is a symlink; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/.gitignore b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/.gitignore new file mode 100644 index 0000000000..d66f42b4eb --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/groups/.gitignore @@ -0,0 +1,3 @@ +# Rendered from ../tasks.json by scripts/render_tasks.py. +* +!.gitignore diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml new file mode 100644 index 0000000000..030258f2e7 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/insights/g1-aggregation.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# FROZEN ANALYST OUTPUT -- do not hand-edit. +# +# Regenerated 2026-08-10 by: +# nemo agents analyst run --workspace smoke-agent --agent smoke-agent \ +# --insights-file-output +# +# over the g1-aggregation/train traces recorded by scripts/record_traces.py: +# 0d8355d6b2af9436b1532c3b5cfcdcee lookup-ada (passing, not referenced) +# 0193ed2c47b54179d201a3a461a6d5fc total-hours-engineers (failing) +# 9815f6448f0b6217ad1f3e124e980128 total-hours-research (failing) +# +# Frozen so the analyst's nondeterminism stays out of a fixture that measures the +# Experimentalist. Regenerating means re-recording the traces first: trace_refs must +# resolve in the target workspace, and the previous version of this file was stale in +# two ways at once -- the ids had expired, and one referenced `total-hours-ops`, which +# the g1 restructure moved into *validation*. +# +# **trace_refs must come from the train split.** Eval Author materializes one task per +# trace, so a validation trace would have it generate a copy of a held-out task and +# quietly undermine the split the fixture depends on. +# +# This file lives under dataset/ because _AGENT_COPY_EXCLUDE_NAMES excludes that +# directory: an Insight describes the weakness in prose, and the Coder must not read it. +insights: +- id: insights-insight-Ax2i7YNQTeDLen9JC9i2cn + workspace: smoke-agent + name: insights-insight-59jp3 + title: Total-hours questions fall through to the fallback answer + agent: smoke-agent + description: When the prompt asks for the total hours for a selected group (for + example, by role or department), smoke-agent's ordered handler dispatch runs handle_lookup, + handle_list, and handle_count, but none of them match the aggregate-total form. + The top-level solve span then returns the fixed fallback string 'I do not know + how to answer that.' instead of writing the required single-line `total=` + answer. This diverges from the contract that sums over records must be reported + with the canonical `total=` key, and suggests the deterministic regex handler + set is missing a total/sum-hours handler for grouped selections. + status: open + trace_refs: + - 0193ed2c47b54179d201a3a461a6d5fc + - 9815f6448f0b6217ad1f3e124e980128 + created_at: '2026-08-10T13:47:28.289641' + updated_at: '2026-08-10T13:47:28.289646' diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md new file mode 100644 index 0000000000..4dd293a017 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/README.md @@ -0,0 +1,47 @@ + + + +# Task template + +Shape for tasks generated from an Insight's production traces. Three placeholders +are filled from the trace: `` and `` in `instruction.md`, and +`` in `tests/expected.txt`. + +## Constraints a generated task must respect + +- **`` must keep the grammar the tasks already use.** A question outside + it fails for the wrong reason -- it looks like the weakness under test but is + really a phrasing miss: + - `What is the of ?` + - `How many people are in the department?` + - `What is the total in the department?` + - `What is the total in the role?` + + The last two say **`in the`**, not `for the`. That is deliberate: the agent's own + `LIST_RE` and `COUNT_RE` prime `in the ... department`, so a total question phrased + with `for` asks the Coder to guess a preposition it was never shown, and the round + then measures luck rather than capability. Scoping by `role` as well as by + `department` is what makes a general fix reachable and a hardcoded one fail + validation. +- **`` is the answer a *correct* agent would give**, keyed by the + canonical record field (`dept`, `role`, `hours`), never the word the question + used. The verifier compares the whole file byte-for-byte. + + **Compute it from `records.json` in this directory.** The trace cannot supply it: + it holds the question and the agent's *wrong* answer, never the right one — that + is what makes it a failure. So read the records, work out what a correct agent + would answer, and write that. A task left with `` in place scores 0 for + every agent, repaired or not, and the run then reads as a failed repair when it + measured nothing at all. + + Worked example: for `What is the total hours in the engineer role?`, sum `hours` + across records whose `role` is `engineer` and write `total=`. +- **Do not edit `tests/test.sh`.** It is synced from `dataset/_shared/` and emits + the `reward` and `shape_ok` keys every task in a dataset must share. +- **Do not add `environment/Dockerfile`.** Tasks reference a prebuilt image via + `[environment].docker_image`; the empty `environment/` directory exists only + because Harbor requires the directory to be present. + +The records available to the agent are at `/app/data/records.json` in the image: +six people across the `research` and `ops` departments, with `name`, `dept`, +`role`, and `hours` fields. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/environment/.gitkeep b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/environment/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md new file mode 100644 index 0000000000..9b116d66c6 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md @@ -0,0 +1,9 @@ + + +Write a single line of text to `/app/artifacts/output.txt` in exactly this form: + +``` += +``` + +No spaces around the `=`, no extra words. diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/records.json b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/records.json new file mode 100644 index 0000000000..10f5cad8e7 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/records.json @@ -0,0 +1,8 @@ +[ + {"name": "Ada Lovelace", "dept": "research", "role": "engineer", "hours": 12}, + {"name": "Grace Hopper", "dept": "research", "role": "engineer", "hours": 8}, + {"name": "Zoë Washington", "dept": "research", "role": "analyst", "hours": 9}, + {"name": "O'Brien", "dept": "ops", "role": "operator", "hours": 5}, + {"name": "Ann-Marie Cruz", "dept": "ops", "role": "operator", "hours": 8}, + {"name": "Karl Jung", "dept": "ops", "role": "", "hours": 0} +] diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml new file mode 100644 index 0000000000..453839f8cd --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/task.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version = "1.1" + +artifacts = [{ source = "/app/artifacts", destination = "output" }] + +[task] +name = "smoke/generated" +authors = [{ name = "NVIDIA" }] +keywords = ["smoke", "g1"] + +[metadata] +difficulty = "easy" +category = "smoke" + +[agent] +timeout_sec = 120.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +# Set by scripts/build_image.py; a test asserts it matches the current content +# hash of the Dockerfile and records file. Tasks ship no environment/ directory. +docker_image = "smoke-agent-env:sha-16a0493fe720" +cpus = 1 +memory_mb = 1024 diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt new file mode 100644 index 0000000000..288c2884a9 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt @@ -0,0 +1 @@ + diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh new file mode 100755 index 0000000000..f17fa04dad --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Harbor copies this directory to /tests and runs this after the agent phase. Its +# only job is to write numeric rewards to /logs/verifier/reward.json. +# +# reward 1.0 when the output matches tests/expected.txt exactly +# shape_ok 1.0 when the first line has the `=` shape, whatever +# the value. This is the discriminating second metric: it separates +# answering the wrong question from not answering at all. Do not +# replace it with a wrote-a-file check — the agent always writes +# one, so that metric would be constant and the ranking 1-D. +# +# Never `set -e`: exiting before reward.json is written turns a legitimate 0 into +# a *missing* metric, which the Experimentalist treats very differently. +set -uo pipefail + +mkdir -p /logs/verifier + +OUTPUT=/app/artifacts/output.txt +EXPECTED_FILE=/tests/expected.txt +reward=0.0 +shape_ok=0.0 + +# Fail closed. With `set -e` off, an unreadable fixture would otherwise leave the +# expectation empty, and empty compares equal to empty — a broken fixture would +# score 1.0. +if [ ! -r "$EXPECTED_FILE" ]; then + echo "FAIL: ${EXPECTED_FILE} is missing or unreadable; refusing to score" +elif [ -L "$OUTPUT" ] || [ -L /app/artifacts ]; then + # The agent owns /app/artifacts and this runs afterwards, so it could replace the + # answer with a symlink at the expected fixture: `-f`, `head`, `sed` and `cmp` all + # follow links, so the answer key would be compared against itself and score 1.0 + # without anything being solved. `-L` is the one test that does not follow, and it + # has to come first for the same reason. The directory is checked too, since + # linking it moves the whole path somewhere else. + # + # This catches symlinks, not hard links -- one of those is indistinguishable from a + # regular file here. It needs both paths on one filesystem, which separate mounts + # prevent, so it is left as a known limit rather than guessed at. + echo "FAIL: ${OUTPUT} is a symlink; refusing to score" +elif [ -f "$OUTPUT" ]; then + # Shape check on the first line only: right form, value not considered. `grep -q` + # is silent on purpose — echoing the line would put answers in the trial log, + # which the Coder can read. + if head -n 1 "$OUTPUT" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*='; then + shape_ok=1.0 + fi + # Byte-for-byte over the whole file. Command substitution would strip trailing + # newlines on both sides, letting an agent append blank lines and still score. + # CRLF is normalized at end-of-line only: `tr -d '\r'` would delete every CR, + # so `sum=42` would collapse into a passing `sum=42`. + EXPECTED_NORM="$(mktemp)" + ACTUAL_NORM="$(mktemp)" + sed 's/\r$//' "$EXPECTED_FILE" > "$EXPECTED_NORM" + sed 's/\r$//' "$OUTPUT" > "$ACTUAL_NORM" + if cmp -s "$EXPECTED_NORM" "$ACTUAL_NORM"; then + reward=1.0 + fi + rm -f "$EXPECTED_NORM" "$ACTUAL_NORM" +else + echo "FAIL: ${OUTPUT} was not created by the agent" +fi + +printf '{"reward": %s, "shape_ok": %s}\n' "$reward" "$shape_ok" > /logs/verifier/reward.json +cat /logs/verifier/reward.json diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/dataset/tasks.json b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/tasks.json new file mode 100644 index 0000000000..973d0b3a6c --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/dataset/tasks.json @@ -0,0 +1,455 @@ +{ + "schema_version": 1, + "tasks": [ + { + "group": "g1-aggregation", + "split": "insight-evidence", + "id": "total-hours-analysts", + "question": "What is the total hours in the analyst role?", + "expected": "total=9", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "insight-evidence", + "id": "total-hours-engineers", + "question": "What is the total hours in the engineer role?", + "expected": "total=20", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "insight-evidence", + "id": "total-hours-operators", + "question": "What is the total hours in the operator role?", + "expected": "total=13", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "insight-evidence", + "id": "total-hours-ops", + "question": "What is the total hours in the ops department?", + "expected": "total=13", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "insight-evidence", + "id": "total-hours-research", + "question": "What is the total hours in the research department?", + "expected": "total=29", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "train", + "id": "lookup-ada", + "question": "What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "train", + "id": "total-hours-engineers", + "question": "What is the total hours in the engineer role?", + "expected": "total=20", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "train", + "id": "total-hours-research", + "question": "What is the total hours in the research department?", + "expected": "total=29", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "validation", + "id": "lookup-grace", + "question": "What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "validation", + "id": "total-hours-analysts", + "question": "What is the total hours in the analyst role?", + "expected": "total=9", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g1-aggregation", + "split": "validation", + "id": "total-hours-ops", + "question": "What is the total hours in the ops department?", + "expected": "total=13", + "format": "total=", + "legacy_environment_comment": false + }, + { + "group": "g2-name-patterns", + "split": "insight-evidence", + "id": "lookup-ann-marie", + "question": "What is the department of Ann-Marie Cruz?", + "expected": "dept=ops", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "insight-evidence", + "id": "lookup-ann-marie-role", + "question": "What is the role of Ann-Marie Cruz?", + "expected": "role=operator", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "insight-evidence", + "id": "lookup-obrien", + "question": "What is the department of O'Brien?", + "expected": "dept=ops", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "insight-evidence", + "id": "lookup-obrien-hours", + "question": "What is the hours of O'Brien?", + "expected": "hours=5", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "insight-evidence", + "id": "lookup-zoe", + "question": "What is the department of Zoë Washington?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "train", + "id": "lookup-ada", + "question": "What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "train", + "id": "lookup-obrien", + "question": "What is the department of O'Brien?", + "expected": "dept=ops", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "train", + "id": "lookup-zoe", + "question": "What is the department of Zoë Washington?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "validation", + "id": "lookup-ann-marie", + "question": "What is the department of Ann-Marie Cruz?", + "expected": "dept=ops", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "validation", + "id": "lookup-grace", + "question": "What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g2-name-patterns", + "split": "validation", + "id": "lookup-role-obrien", + "question": "What is the role of O'Brien?", + "expected": "role=operator", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "insight-evidence", + "id": "preamble-dept", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "insight-evidence", + "id": "preamble-dept-zoe", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Zoë Washington?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "insight-evidence", + "id": "preamble-hours-obrien", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the hours of O'Brien?", + "expected": "hours=5", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "insight-evidence", + "id": "preamble-role", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the role of Ada Lovelace?", + "expected": "role=engineer", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "insight-evidence", + "id": "preamble-role-grace", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the role of Grace Hopper?", + "expected": "role=engineer", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "train", + "id": "plain-dept", + "question": "What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "train", + "id": "preamble-dept", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "train", + "id": "preamble-role", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the role of Ada Lovelace?", + "expected": "role=engineer", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "validation", + "id": "preamble-hours", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the hours of Grace Hopper?", + "expected": "hours=8", + "format": "hours=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "validation", + "id": "preamble-long-dept", + "question": "Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g3-long-inputs", + "split": "validation", + "id": "trailing-prose", + "question": "What is the department of Grace Hopper? Reporting policy note. All figures in this report are drawn from the current staffing register and are considered provisional until the quarterly review has signed them off. Where a value is disputed, the register takes precedence over any earlier summary. Do not round, abbreviate, or reformat values when reporting them. ", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "train", + "id": "count-ops", + "question": "How many people are in the ops department?", + "expected": "count=3", + "format": "count=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "train", + "id": "count-research", + "question": "How many people are in the research department?", + "expected": "count=3", + "format": "count=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "train", + "id": "lookup-ada", + "question": "What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "validation", + "id": "count-engineers-research", + "question": "How many engineers are in the research department?", + "expected": "count=2", + "format": "count=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "validation", + "id": "count-operators-ops", + "question": "How many operators are in the ops department?", + "expected": "count=2", + "format": "count=", + "legacy_environment_comment": true + }, + { + "group": "g4-dispatch-order", + "split": "validation", + "id": "lookup-grace", + "question": "What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "insight-evidence", + "id": "empty-role", + "question": "What is the role of Karl Jung?", + "expected": "role=unknown", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "insight-evidence", + "id": "missing-person", + "question": "What is the department of Alan Turing?", + "expected": "dept=unknown", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "insight-evidence", + "id": "missing-person-katherine", + "question": "What is the department of Katherine Johnson?", + "expected": "dept=unknown", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "insight-evidence", + "id": "missing-person-linus", + "question": "What is the department of Linus Torvalds?", + "expected": "dept=unknown", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "insight-evidence", + "id": "missing-person-marie", + "question": "What is the department of Marie Curie?", + "expected": "dept=unknown", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "train", + "id": "empty-role", + "question": "What is the role of Karl Jung?", + "expected": "role=unknown", + "format": "role=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "train", + "id": "lookup-ada", + "question": "What is the department of Ada Lovelace?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "train", + "id": "missing-person", + "question": "What is the department of Alan Turing?", + "expected": "dept=unknown", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "validation", + "id": "lookup-grace", + "question": "What is the department of Grace Hopper?", + "expected": "dept=research", + "format": "dept=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "validation", + "id": "missing-person-hours", + "question": "What is the hours of Alan Turing?", + "expected": "hours=unknown", + "format": "hours=", + "legacy_environment_comment": true + }, + { + "group": "g5-edge-cases", + "split": "validation", + "id": "missing-person-role", + "question": "What is the role of Alan Turing?", + "expected": "role=unknown", + "format": "role=", + "legacy_environment_comment": true + } + ] +} diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-full.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-full.yaml new file mode 100644 index 0000000000..3638b5efe0 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-full.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Full smoke scenario. `scripts/build_all_group.py` generates these datasets +# from the supported repair groups before this profile is used. + +agent: smoke-agent +agent_source: ./agent +agent_spec: ./AGENT-SPEC.md +task_template: ./dataset/task-template + +datasets: + train: ./dataset/groups/_all/train + validation: ./dataset/groups/_all/validation + +workspace: default diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml new file mode 100644 index 0000000000..8d4c381d2f --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer-generalization.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The generalization scenario's profile. Identical to optimizer.yaml except for +# the datasets, which is the whole point: what separates a repair scenario from a +# generalization one is the split, not any tuning knob. +# +# nemo agents experimentalist run --profile optimizer-generalization.yaml \ +# --no-insight --config configs/short.yaml +# +# g4-dispatch-order's train split shows only plain department counts, so the +# reachable fix -- reordering the handler chain -- passes train. Validation asks +# for role-scoped counts, which that fix does not reach. A healthy run therefore +# *keeps the baseline*, the opposite of what the repair scenario asserts. +# +# This profile exists because a config alone could not express that. Both +# scenarios use configs/short.yaml; optimizer.yaml pins the repair datasets to +# g1-aggregation, while this profile selects the generalization split. + +# Logical agent name. In insight mode it must match the Insight's agent; under +# --no-insight it is only a label. +agent: smoke-agent + +# Same agent under test as the repair scenario. Only the tasks differ. +agent_source: ./agent + +agent_spec: ./AGENT-SPEC.md + +# Required by the profile schema even under --no-insight, where it is never read. +task_template: ./dataset/task-template + +datasets: + train: ./dataset/groups/g4-dispatch-order/train + validation: ./dataset/groups/g4-dispatch-order/validation + +workspace: default diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml new file mode 100644 index 0000000000..4a3b376343 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/optimizer.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Smoke fixture profile. Everything is local: a "./" prefix classifies as a local +# path, so no registry_url is needed and nothing is downloaded. +# +# nemo agents experimentalist doctor --profile +# nemo agents experimentalist run --profile --no-insight + +# Logical agent name. In insight mode it must match the Insight's agent; under +# --no-insight it is only a label. +agent: smoke-agent + +# Where the baseline agent's code lives, relative to this file. The whole +# directory is copied into every candidate workspace and is readable by the +# Coder, so it holds the agent and nothing else: no README, no configs, no +# scripts, no dataset, no .env. Anything that should not reach the Coder simply +# stays out of this folder rather than relying on an exclusion list. +agent_source: ./agent + +# Behaviour contract threaded to the analyzer and goal-tree components. It states +# the rules the agent must keep, chiefly that it stays offline and reproducible. +agent_spec: ./AGENT-SPEC.md + +# Required by the profile schema even under --no-insight, where it is never read. +# It points at a single Harbor task directory used as the shape for Eval Author +# generated tasks in insight mode. +task_template: ./dataset/task-template + +datasets: + train: ./dataset/groups/g1-aggregation/train + validation: ./dataset/groups/g1-aggregation/validation + +workspace: default diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py new file mode 100644 index 0000000000..0ce35f8360 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_all_group.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assemble `dataset/groups/_all/` from the groups listed in `source_groups`. + +The loop takes one train/validation pair, so exercising several groups in a +single run means one combined dataset. Directory names collide -- every group has +a `lookup-ada` or `lookup-grace` control -- so each is prefixed with its group +key. The `[task] name` values inside were already unique. + +Generated, not authored: rerun this after changing any group. A test asserts the +combined copy matches its sources, so a stale `_all/` fails rather than silently +running against old tasks. +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +COMBINED = "_all" +SPLITS = ("train", "validation") + +# Not part of the combined set. Both are held out because the combined scenario +# asserts that every task in it is reachable, and neither of these is. +# +# `g4-dispatch-order` backs the generalization scenario, which asserts the +# opposite outcome: there, retaining the baseline is a pass. One run cannot +# assert both. +# +# `g5-edge-cases` is reachable only when trajectory scoring is on, and the +# combined scenario runs with it off. Measured across runs made after the spec +# stated the sentinel, so the spec is not the variable: +# +# goal tree off -> 0 of 13 candidates closed g5 +# goal tree on -> 7 of 11 candidates closed g5 +# +# The pattern is that the goal tree sharpens the analysis enough for the Coder to +# see both halves of the fix; without it the analysis names one half and the +# candidates fix one half, which scores nothing. Trajectory scoring is off here +# because it is not yet dependable -- it has silently skipped a candidate, and a +# rejected goal tree disables it for a whole run without saying so. +# +# PUT G5 BACK once trajectory scoring is dependable enough to leave on. It is the +# only group that exercises a fix needing several coordinated edits, so the +# combined scenario is weaker without it. +EXCLUDED_GROUPS = frozenset({"g4-dispatch-order", "g5-edge-cases"}) + + +def group_key(group: str) -> str: + """Return the short key a group's task names already use (``g1`` from ``g1-aggregation``).""" + return group.split("-")[0] + + +def source_groups(dataset_dir: Path) -> list[str]: + """Every group the combined set is built from, in a stable order.""" + groups = dataset_dir / "groups" + return sorted( + d.name for d in groups.iterdir() if d.is_dir() and d.name != COMBINED and d.name not in EXCLUDED_GROUPS + ) + + +def assemble(dataset_dir: Path) -> list[Path]: + """Rebuild the combined group from its sources; return the task directories written.""" + target = dataset_dir / "groups" / COMBINED + shutil.rmtree(target, ignore_errors=True) + + written: list[Path] = [] + for group in source_groups(dataset_dir): + key = group_key(group) + for split in SPLITS: + for task in sorted((dataset_dir / "groups" / group / split).iterdir()): + if not (task / "task.toml").is_file(): + continue + dest = target / split / f"{key}-{task.name}" + shutil.copytree(task, dest) + written.append(dest) + return written + + +def main() -> None: + """Assemble the combined group from the command line.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + args = parser.parse_args() + from render_tasks import render + + render(args.dataset_dir) + written = assemble(args.dataset_dir) + for path in written: + print(path.relative_to(args.dataset_dir)) + print(f"{len(written)} tasks") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py new file mode 100644 index 0000000000..9b9d8dacf3 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/build_image.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the shared task image and render the task directories. + +The tag is a content hash of the Dockerfile and the records file, so a change to +either produces a new tag. Tasks reference the tag rather than carrying their own +Dockerfile, and a test asserts every task references the current one -- which is +what turns "forgot to rebuild" into a failing test instead of a container quietly +running against stale data. +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +from pathlib import Path + +from render_tasks import render + +HASHED_FILES = ("Dockerfile", "records.json") +IMAGE_NAME = "smoke-agent-env" +_DOCKER_IMAGE_RE = re.compile(r'^(docker_image\s*=\s*)"[^"]*"', re.MULTILINE) + + +def image_tag(shared_dir: Path) -> str: + """Return the content-addressed tag for the current shared assets.""" + digest = hashlib.sha256() + for name in HASHED_FILES: + digest.update((shared_dir / name).read_bytes()) + return f"{IMAGE_NAME}:sha-{digest.hexdigest()[:12]}" + + +def build(shared_dir: Path, tag: str) -> None: + """Build the image. Docker layer caching makes a no-op rebuild cheap.""" + subprocess.run(["docker", "build", "-t", tag, str(shared_dir)], check=True) + + +def task_tomls(dataset_dir: Path) -> list[Path]: + """Return task manifests that are contained in the rendered dataset.""" + root = dataset_dir.resolve(strict=True) + manifests: list[Path] = [] + for task_root in (root / "task-template", root / "groups"): + if task_root.is_symlink() or not task_root.is_dir(): + raise ValueError(f"expected a real task directory under {root}: {task_root.name}") + for task_toml in sorted(task_root.rglob("task.toml")): + parent = task_toml.parent.resolve(strict=True) + if task_toml.is_symlink() or not parent.is_relative_to(root): + raise ValueError(f"task manifest escapes the dataset: {task_toml}") + manifests.append(task_toml) + return manifests + + +def ensure_environment_dirs(dataset_dir: Path) -> list[Path]: + """Create the empty environment/ every task needs; return the ones created. + + Harbor's ``TaskModel.is_valid_dir`` requires ``environment/`` to *exist* before + it will even parse a task; ``[environment].docker_image`` only makes the + Dockerfile inside it optional. A task without the directory is silently not a + task -- the dataset loads with zero tasks rather than erroring. The directory + stays empty apart from a .gitkeep, since a Dockerfile there would shadow the + prebuilt image. + """ + created: list[Path] = [] + for task_toml in task_tomls(dataset_dir): + environment = task_toml.parent / "environment" + if environment.is_symlink(): + raise ValueError(f"task environment escapes the dataset: {environment}") + keep = environment / ".gitkeep" + if not keep.exists(): + keep.parent.mkdir(parents=True, exist_ok=True) + keep.touch() + created.append(keep) + return created + + +def stamp_tasks(dataset_dir: Path, tag: str) -> list[Path]: + """Rewrite every task.toml's docker_image to *tag*; return the ones changed.""" + changed: list[Path] = [] + for task_toml in task_tomls(dataset_dir): + text = task_toml.read_text(encoding="utf-8") + updated = _DOCKER_IMAGE_RE.sub(rf'\1"{tag}"', text) + if updated != text: + task_toml.write_text(updated, encoding="utf-8") + changed.append(task_toml) + return changed + + +def main() -> None: + """Build the shared image and render every curated task.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + parser.add_argument("--skip-build", action="store_true", help="stamp tags without invoking docker") + args = parser.parse_args() + + render(args.dataset_dir) + tag = image_tag(args.dataset_dir / "_shared") + if not args.skip_build: + build(args.dataset_dir / "_shared", tag) + for path in ensure_environment_dirs(args.dataset_dir): + print(f"created {path}") + for path in stamp_tasks(args.dataset_dir, tag): + print(f"stamped {path}") + print(tag) + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py new file mode 100644 index 0000000000..018e014d60 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Evaluate one group's tasks and upload their traces to Intake. + +Insight mode needs an Insight whose trace_refs resolve through the Platform +client, so a group's own tasks double as the trace source: evaluate them, then +ingest the trace each trial wrote. + +The resource attributes attached here are not decoration. `gen_ai.agent.name` is +how the analyst later finds these traces, so the Insight generated from them is +only possible because they are set. +""" + +from __future__ import annotations + +import argparse +import asyncio +from pathlib import Path + +from nemo_experimentalist_plugin.client import make_client +from nemo_experimentalist_plugin.entities import TrialResult, local_path_from_uri +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import ( + HarborDataset, + HarborEvaluator, + HarborEvaluatorConfig, +) +from nemo_experimentalist_plugin.experimentalist.otlp import jsonl_to_protobuf, read_trace_id +from nemo_platform import AsyncNeMoPlatform, NotFoundError + +AGENT_NAME = "smoke-agent" +AGENT_VERSION = "1.0.0" +INGEST_PATH = "/apis/intake/v2/workspaces/{workspace}/ingest/otlp/v1/traces" +POLL_ATTEMPTS = 30 +POLL_DELAY_SECONDS = 2.0 + + +async def _upload_trials( + client: AsyncNeMoPlatform, + trials: list[TrialResult], + *, + workspace: str, + group: str, +) -> dict[str, str]: + """Upload each trial's trace; return {task_id: trace_id}.""" + url = INGEST_PATH.format(workspace=workspace) + trace_ids: dict[str, str] = {} + + for trial in trials: + if trial.trace is None: + raise RuntimeError(f"Trial {trial.id} produced no trace; the agent must write /app/traces") + trace_id = read_trace_id(trial.trace) + if trace_id in trace_ids.values(): + raise RuntimeError(f"Trace {trace_id} was produced by more than one trial") + + attrs = { + "nemo.experiment.id": group, + "nemo.test_case.id": trial.task_id, + "nemo.trial.id": trial.id, + "gen_ai.agent.name": AGENT_NAME, + "gen_ai.agent.version": AGENT_VERSION, + } + path = local_path_from_uri(trial.trace.uri, context="Agent execution trace") + payloads = jsonl_to_protobuf(path, extra_resource_attrs=attrs) + if not payloads: + raise RuntimeError(f"Trial {trial.id} produced an empty trace") + for payload in payloads: + await client.post( + url, + cast_to=object, + content=payload, + options={"headers": {"Content-Type": "application/x-protobuf"}}, + ) + # Every recording run uses one attempt per task. Keep the task id here + # so the caller can put precisely the failing task traces in an Insight. + trace_ids[trial.task_id] = trace_id + + return trace_ids + + +async def _wait_retrievable(client: AsyncNeMoPlatform, workspace: str, trace_ids: set[str]) -> None: + """Block until every trace id resolves, or raise once the budget is spent.""" + pending = set(trace_ids) + for _ in range(POLL_ATTEMPTS): + for trace_id in sorted(pending): + try: + await client.intake.traces.retrieve(trace_id, workspace=workspace) + except NotFoundError: + continue + pending.discard(trace_id) + if not pending: + return + await asyncio.sleep(POLL_DELAY_SECONDS) + raise TimeoutError(f"traces never became retrievable: {sorted(pending)}") + + +async def run(args: argparse.Namespace) -> dict[str, str]: + """Evaluate the group's train split, then upload every trial's trace.""" + agent_path = args.agent.expanduser().resolve() + dataset_path = (args.dataset_root / "groups" / args.group / args.split).expanduser().resolve() + if not dataset_path.is_dir(): + raise FileNotFoundError(f"group dataset not found: {dataset_path}") + + dataset = HarborDataset.from_path(dataset_path) + run_dir = args.output.expanduser().resolve() / args.group / args.split + if run_dir.exists(): + raise FileExistsError(f"output directory already exists: {run_dir}") + run_dir.mkdir(parents=True) + + client = make_client(args.base_url) + try: + await client.workspaces.create( + name=args.workspace, + description="smoke-agent traces for Insights", + exist_ok=True, + ) + options = HarborEvaluatorConfig( + job_name=f"smoke-{args.group}-{args.split}-record", + jobs_dir=Path("results"), + n_attempts=1, + n_concurrent_trials=args.concurrency, + quiet=True, + ) + result = await HarborEvaluator(experiment_dir=run_dir).run( + agent=agent_path, + dataset=dataset, + options=options, + ) + trials = list(result.trials) + if not trials: + raise RuntimeError(f"no trials produced under {run_dir}") + + trace_ids = await _upload_trials(client, trials, workspace=args.workspace, group=args.group) + await _wait_retrievable(client, args.workspace, set(trace_ids.values())) + return trace_ids + finally: + await client.close() + + +def main() -> None: + """Record and ingest one group's traces.""" + example_dir = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser() + parser.add_argument("--group", required=True) + parser.add_argument("--split", default="train") + parser.add_argument("--workspace", default="smoke-agent") + parser.add_argument("--agent", type=Path, default=example_dir) + parser.add_argument("--dataset-root", type=Path, default=example_dir / "dataset") + parser.add_argument("--output", type=Path, default=Path("tmp/smoke-record")) + parser.add_argument("--base-url", default=None) + parser.add_argument("--concurrency", type=int, default=3) + args = parser.parse_args() + + for task_id, trace_id in asyncio.run(run(args)).items(): + print(f"{task_id} {trace_id}") + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/render_tasks.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/render_tasks.py new file mode 100644 index 0000000000..0849eb2b29 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/render_tasks.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render the compact smoke-task manifest into Harbor task directories.""" + +from __future__ import annotations + +import argparse +import json +import shutil +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class TaskSpec: + """The authored values for one curated task.""" + + group: str + split: str + id: str + question: str + expected: str + format: str + legacy_environment_comment: bool + + @property + def name(self) -> str: + """Return the Harbor task name.""" + return f"smoke/{self.group.split('-', 1)[0]}-{self.id}" + + +def load_tasks(dataset_dir: Path) -> list[TaskSpec]: + """Load the compact curated-task manifest.""" + payload = json.loads((dataset_dir / "tasks.json").read_text(encoding="utf-8")) + entries = payload.get("tasks") + if not isinstance(entries, list): + raise ValueError("tasks.json must contain a tasks list") + tasks: list[TaskSpec] = [] + for entry in entries: + if not isinstance(entry, dict): + raise ValueError("each task must be an object") + try: + task = TaskSpec( + group=str(entry["group"]), + split=str(entry["split"]), + id=str(entry["id"]), + question=str(entry["question"]), + expected=str(entry["expected"]), + format=str(entry["format"]), + legacy_environment_comment=bool(entry["legacy_environment_comment"]), + ) + except KeyError as exc: + raise ValueError(f"task is missing {exc.args[0]!r}") from exc + if not task.group or not task.split or not task.id or not task.question or not task.expected: + raise ValueError(f"task {task!r} has an empty required value") + if "=" not in task.expected: + raise ValueError(f"task {task.id!r} expected value has no key") + tasks.append(task) + if len({(task.group, task.split, task.id) for task in tasks}) != len(tasks): + raise ValueError("tasks.json contains duplicate group/split/id entries") + return tasks + + +def render(dataset_dir: Path) -> list[Path]: + """Render every curated task from ``task-template`` and return their paths.""" + template = dataset_dir / "task-template" + groups = dataset_dir / "groups" + if not template.is_dir(): + raise FileNotFoundError(f"task template not found: {template}") + groups.mkdir(exist_ok=True) + for path in groups.iterdir(): + if path.name == ".gitignore": + continue + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + written: list[Path] = [] + for task in load_tasks(dataset_dir): + destination = groups / task.group / task.split / task.id + shutil.copytree(template, destination, ignore=shutil.ignore_patterns("README.md", "records.json")) + _render_task(destination, task) + written.append(destination) + return written + + +def _render_task(destination: Path, task: TaskSpec) -> None: + """Fill one copied task template.""" + replacements = { + "": task.question, + "": task.format.partition("=")[0], + "": task.expected, + 'name = "smoke/generated"': f'name = "{task.name}"', + 'keywords = ["smoke", "g1"]': f'keywords = ["smoke", "{task.group.split("-", 1)[0]}"]', + } + for relative in ("instruction.md", "task.toml", "tests/expected.txt"): + path = destination / relative + text = path.read_text(encoding="utf-8") + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + if relative == "instruction.md": + text = text.replace(f"{task.format.partition('=')[0]}=", task.format) + if relative == "task.toml" and task.legacy_environment_comment: + text = text.replace( + "Tasks ship no environment/ directory.", + "environment/ stays empty: Harbor\n# requires the directory, and a Dockerfile would shadow the prebuilt image.", + ) + remaining = [placeholder for placeholder in ("", "", "") if placeholder in text] + if remaining: + raise ValueError(f"unfilled placeholder in {path}: {remaining}") + path.write_text(text, encoding="utf-8") + + +def main() -> None: + """Render curated tasks from the command line.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "dataset", + ) + args = parser.parse_args() + for path in render(args.dataset_dir): + print(path.relative_to(args.dataset_dir)) + + +if __name__ == "__main__": + main() diff --git a/plugins/nemo-experimentalist/pyproject.toml b/plugins/nemo-experimentalist/pyproject.toml index 842f9ff161..48ac1bdd29 100644 --- a/plugins/nemo-experimentalist/pyproject.toml +++ b/plugins/nemo-experimentalist/pyproject.toml @@ -35,3 +35,6 @@ packages = ["src/nemo_experimentalist_plugin"] asyncio_mode = "auto" pythonpath = ["src"] testpaths = ["tests"] +markers = [ + "e2e: executes model-written shell and requires a sandbox, Docker, a running Platform, and configured models", +] diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/factory.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/factory.py index cc800b57f3..28e58d6656 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/factory.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/factory.py @@ -33,7 +33,7 @@ def __init__( ) -> None: self.supported_evaluator_types = supported_evaluator_types or _SUPPORTED_EVALUATOR_TYPES - def build_dataset(self, evaluator_type: EvaluatorType, dataset_ref: DatasetRef) -> Dataset: + def build_dataset(self, evaluator_type: EvaluatorType, dataset_ref: DatasetRef, **options: Any) -> Dataset: """Build a Dataset for the selected evaluator type. Args: @@ -51,7 +51,7 @@ def build_dataset(self, evaluator_type: EvaluatorType, dataset_ref: DatasetRef) if evaluator_type not in self.supported_evaluator_types: raise ValueError(f"Unsupported evaluator type: {evaluator_type}") - return self.supported_evaluator_types[evaluator_type][0].from_ref(dataset_ref) + return self.supported_evaluator_types[evaluator_type][0].from_ref(dataset_ref, **options) def build_task_template(self, evaluator_type: EvaluatorType, template_ref: DatasetRef) -> Task: """Parse an evaluator-specific template directory as one task. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py index bdbd1244fd..cfcbf609fe 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py @@ -952,7 +952,7 @@ def __init__( ) @classmethod - def from_ref(cls, ref: DatasetRef, **options: Any) -> HarborDataset: + def from_ref(cls, ref: DatasetRef, *, allow_empty: bool = False, **options: Any) -> HarborDataset: """Build a Harbor dataset from a local dataset reference.""" dataset_path = local_path_from_uri(ref.uri, context="Harbor dataset reference") dataset_id = ref.metadata.get("id") @@ -962,6 +962,7 @@ def from_ref(cls, ref: DatasetRef, **options: Any) -> HarborDataset: dataset = cls.from_path( dataset_path, dataset_id=dataset_id, + allow_empty=allow_empty, **options, ) return dataset.subset(task_ids) if task_ids is not None else dataset @@ -985,6 +986,7 @@ def from_path( dataset_path: Path, *, dataset_id: str | None = None, + allow_empty: bool = False, **_ignored_options: Any, ) -> HarborDataset: """Build a Harbor dataset from a local Harbor task collection.""" @@ -995,7 +997,7 @@ def from_path( raise ValueError(f"Harbor dataset path is not a directory: {dataset_path}") task_dirs = cls._find_task_dirs(dataset_path) - if not task_dirs: + if not task_dirs and not allow_empty: raise ValueError(f"Harbor dataset path contains no Harbor task directories: {dataset_path}") tasks = [cls._from_task_dir(task_dir) for task_dir in task_dirs] diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index c363581eda..38d0073091 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -526,11 +526,13 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: train_eval_dataset = dataset_factory.build_dataset( deps.evaluator_type, train_dataset_ref, + allow_empty=deps.insight is not None, ) validation_eval_dataset = dataset_factory.build_dataset( deps.evaluator_type, validation_dataset_ref, + allow_empty=deps.insight is not None, ) # ---- Resolve insight (Mode 1) vs local agent (Mode 2) ----------- @@ -808,6 +810,9 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: phase=phase, config=config, ) + if not improvements: + logger.info("[TERMINATOR] no improvements proposed; finalizing evaluated candidates") + break new_candidates = [ self._create_agent( agents_dir=agents_dir, diff --git a/plugins/nemo-experimentalist/tests/conftest.py b/plugins/nemo-experimentalist/tests/conftest.py index 292f4c3f34..a11842d417 100644 --- a/plugins/nemo-experimentalist/tests/conftest.py +++ b/plugins/nemo-experimentalist/tests/conftest.py @@ -3,7 +3,15 @@ """Experimentalist test-wide state isolation.""" +from __future__ import annotations + import os +import shlex +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path from typing import cast import litellm @@ -15,6 +23,241 @@ # Drop unsupported params silently so the CodeAct strategy can call tools. litellm.drop_params = True +_REPO_ROOT = Path(__file__).resolve().parents[3] +_PLUGIN_ROOT = _REPO_ROOT / "plugins" / "nemo-experimentalist" +_CI_ENV = "CI" +_SANDBOX_NAME_ENV = "SANDBOX_VM_ID" +_ALLOW_UNSANDBOXED_ENV = "SMOKE_AGENT_E2E_ALLOW_UNSANDBOXED" + + +def pytest_runtest_setup(item: pytest.Item) -> None: + """Check that marked E2E tests are isolated.""" + if "e2e" not in item.keywords: + return + if os.environ.get(_CI_ENV): + pytest.skip("smoke-agent E2E tests are developer-invoked and do not run in CI") + if os.environ.get(_SANDBOX_NAME_ENV) or os.environ.get(_ALLOW_UNSANDBOXED_ENV) == "1": + return + pytest.skip( + "e2e executes model-written shell; set SANDBOX_VM_ID to an existing sbx sandbox " + "or set SMOKE_AGENT_E2E_ALLOW_UNSANDBOXED=1" + ) + + +class SandboxRunner: + """Run and fetch smoke-agent work.""" + + def __init__(self, sandbox: str | None) -> None: + self.sandbox = sandbox + self.repo_root = _REPO_ROOT + self.plugin_root = _PLUGIN_ROOT + self.run_root = "/tmp/nemo-experimentalist-smoke-e2e/source" + self.remote_plugin_root = f"{self.run_root}/nemo-experimentalist" + + @property + def platform_url(self) -> str: + """Return the loop's Platform URL.""" + return "http://host.docker.internal:8080" if self.sandbox else "http://localhost:8080" + + def _sandbox_command(self, command: list[str], environment: dict[str, str] | None) -> list[str]: + """Wrap one command for sbx.""" + if not self.sandbox: + return command + wrapped = ["sbx", "exec"] + settings = { + "UV_PROJECT_ENVIRONMENT": "/home/agent/.venvs/nemo-platform", + "PYTHONPATH": f"{self.remote_plugin_root}/src", + "OTLP_ENDPOINT": "http://host.docker.internal:5001/v1/traces", + **(environment or {}), + } + for name, value in settings.items(): + wrapped.extend(["--env", f"{name}={value}"]) + return [*wrapped, "--workdir", str(self.repo_root), self.sandbox, *command] + + def run( + self, + command: list[str], + *, + log: Path, + environment: dict[str, str] | None = None, + capture_output: bool = False, + ) -> str: + """Run a command and write its output to the host log.""" + actual = self._sandbox_command(command, environment) + process_environment = None if self.sandbox else environment + with log.open("a", encoding="utf-8") as output: + output.write("$ " + " ".join(shlex.quote(part) for part in actual) + "\n") + output.flush() + result = subprocess.run( + actual, + cwd=self.repo_root, + env=process_environment, + stdout=subprocess.PIPE if capture_output else output, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if capture_output: + output.write(result.stdout or "") + if result.returncode: + pytest.fail(f"E2E command failed; log: {log}\n{log.read_text(encoding='utf-8')}") + return result.stdout or "" + + def sync(self, *, log: Path) -> None: + """Copy the current plugin tree into the sandbox.""" + if not self.sandbox: + return + self.run(["mkdir", "-p", self.run_root], log=log) + staging_root = Path(tempfile.mkdtemp(prefix="smoke-agent-e2e-sync-")) + staged_plugin = staging_root / self.plugin_root.name + try: + shutil.copytree(self.plugin_root, staged_plugin, ignore=shutil.ignore_patterns("tmp")) + result = subprocess.run( + ["sbx", "cp", str(staged_plugin), f"{self.sandbox}:{self.run_root}"], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + with log.open("a", encoding="utf-8") as output: + output.write(f"$ sbx cp {staged_plugin} {self.sandbox}:{self.run_root}\n") + output.write(result.stdout or "") + output.write(result.stderr or "") + if result.returncode: + pytest.fail( + f"could not sync the Experimentalist worktree; log: {log}\n{log.read_text(encoding='utf-8')}" + ) + finally: + shutil.rmtree(staging_root, ignore_errors=True) + ownership = subprocess.run( + ["sbx", "exec", "--user", "root", self.sandbox, "chown", "-R", "agent:agent", self.remote_plugin_root], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + with log.open("a", encoding="utf-8") as output: + output.write(f"$ sbx exec --user root {self.sandbox} chown -R agent:agent {self.remote_plugin_root}\n") + output.write(ownership.stdout or "") + output.write(ownership.stderr or "") + if ownership.returncode: + pytest.fail(f"could not prepare the synced plugin tree; log: {log}\n{log.read_text(encoding='utf-8')}") + + def prepare_fixture(self, artifact_parent: Path, *, log: Path) -> tuple[str, str]: + """Create one isolated fixture copy.""" + if not self.sandbox: + fixture = artifact_parent / "workspace" / "smoke-agent" + fixture.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(self.plugin_root / "examples" / "smoke-agent", fixture) + return str(fixture), str(artifact_parent / "experiment") + remote_parent = f"{self.run_root}/{artifact_parent.name}-{uuid.uuid4().hex}" + remote_fixture = f"{remote_parent}/workspace/smoke-agent" + self.run( + [ + "sh", + "-lc", + f"mkdir -p {shlex.quote(remote_parent + '/workspace')} && cp -a " + f"{shlex.quote(self.remote_plugin_root + '/examples/smoke-agent')} {shlex.quote(remote_fixture)}", + ], + log=log, + ) + return remote_fixture, f"{remote_parent}/experiment" + + def source_path(self, path: Path) -> str: + """Return a source path visible to the loop.""" + if not self.sandbox: + return str(path) + return str(Path(self.remote_plugin_root) / path.relative_to(self.plugin_root)) + + def replace_text(self, path: str, old: str, new: str, *, log: Path) -> None: + """Edit a fixture file owned by this test.""" + if not self.sandbox: + local = Path(path) + local.write_text(local.read_text(encoding="utf-8").replace(old, new), encoding="utf-8") + return + script = "import pathlib, sys; path=pathlib.Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))" + self.run(["python3", "-c", script, path, old, new], log=log) + + def make_directories(self, *paths: str, log: Path) -> None: + """Create fixture directories.""" + if self.sandbox: + self.run(["mkdir", "-p", *paths], log=log) + else: + for path in paths: + Path(path).mkdir(parents=True, exist_ok=True) + + def copy_in(self, source: Path, destination: str, *, log: Path) -> None: + """Copy a host file into the sandbox.""" + if not self.sandbox: + return + result = subprocess.run( + ["sbx", "cp", str(source), f"{self.sandbox}:{destination}"], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + with log.open("a", encoding="utf-8") as output: + output.write(f"$ sbx cp {source} {self.sandbox}:{destination}\n") + output.write(result.stdout or "") + output.write(result.stderr or "") + if result.returncode: + pytest.fail(f"could not copy E2E input into sandbox; log: {log}\n{log.read_text(encoding='utf-8')}") + + def fetch(self, remote_path: str, local_parent: Path, *, log: Path) -> None: + """Download a sandbox artifact into pytest's directory.""" + if not self.sandbox: + return + local_parent.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ["sbx", "cp", f"{self.sandbox}:{remote_path}", str(local_parent)], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + with log.open("a", encoding="utf-8") as output: + output.write(f"$ sbx cp {self.sandbox}:{remote_path} {local_parent}\n") + output.write(result.stdout or "") + output.write(result.stderr or "") + if result.returncode: + pytest.fail(f"could not fetch sandbox E2E artifacts; log: {log}\n{log.read_text(encoding='utf-8')}") + + +def pytest_sessionstart(session: pytest.Session) -> None: + """Copy the plugin tree once before E2E workers start.""" + if hasattr(session.config, "workerinput") or "e2e" not in session.config.option.markexpr: + return + sandbox = os.environ.get(_SANDBOX_NAME_ENV) + if sandbox is None or os.environ.get(_CI_ENV): + return + sandboxes = subprocess.run( + ["sbx", "ls", "--quiet"], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + if sandboxes.returncode or sandbox not in sandboxes.stdout.splitlines(): + raise pytest.UsageError( + f"sandbox {sandbox!r} does not exist. Create it before running E2E tests:\n" + " sbx create --clone " + f'--name {sandbox} shell "$(git rev-parse --show-toplevel)"\n\n' + "Then run:\n" + f" SANDBOX_VM_ID={sandbox} uv run --frozen pytest " + "plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_1_loop_e2e.py " + "plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_2_loop_e2e.py " + "-m e2e -n 4 --dist loadgroup" + ) + runtime = SandboxRunner(sandbox) + runtime.sync(log=Path("/tmp") / f"smoke-agent-e2e-sync-{uuid.uuid4().hex}.log") + + +@pytest.fixture(scope="session") +def sandbox_runner() -> SandboxRunner: + """Provide the configured sandbox runner.""" + return SandboxRunner(os.environ.get(_SANDBOX_NAME_ENV)) + @pytest.fixture(autouse=True) def _restore_environ(): diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py index 3d91189e2b..4ad8ffeef8 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py @@ -66,6 +66,16 @@ def test_harbor_dataset_add_tasks_copies_task_directories(tmp_path: Path) -> Non assert Path(train_dataset.get_task("insight-task").uri.removeprefix("file://")) == destination_root / "insight-task" +def test_harbor_dataset_allows_an_empty_split_only_when_requested(tmp_path: Path) -> None: + """Check that an empty split is accepted only for an Insight-generated suite.""" + with pytest.raises(ValueError, match="contains no Harbor task directories"): + HarborDataset.from_path(tmp_path) + + dataset = HarborDataset.from_path(tmp_path, allow_empty=True) + + assert dataset.list_tasks() == [] + + def test_harbor_dataset_add_tasks_preserves_existing_task_when_replacement_is_invalid(tmp_path: Path) -> None: source_root = tmp_path / "insight-suite" destination_root = tmp_path / "train" diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py index fcb76a613b..6d2404c76c 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py @@ -43,7 +43,7 @@ def __init__(self, train_dataset: SimpleNamespace, validation_dataset: SimpleNam self.metric_keys = ("reward",) class RecordingDatasetFactory: - def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: + def build_dataset(self, evaluator_type: str, ref: DatasetRef, **_options: object) -> SimpleNamespace: return SimpleNamespace(ref=ref) def build_task_template(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_failure.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_failure.py index cefb0e3ae3..66d0cb2fe5 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_failure.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_failure.py @@ -8,9 +8,11 @@ import pytest from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig -from nemo_experimentalist_plugin.entities import Candidate, ExperimentRun +from nemo_experimentalist_plugin.entities import Candidate, EvaluationResult, ExperimentRun from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer +from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionTree +from nemo_experimentalist_plugin.experimentalist.components.terminator import TerminationDecision from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import LocalExperimentalistBackend @@ -127,3 +129,88 @@ async def run(self, *args, **kwargs): candidates=[candidate], config=EvolutionaryOptimizerConfig(), ) + + +@pytest.mark.asyncio +async def test_no_proposals_finalizes_evaluated_candidates(monkeypatch, tmp_path): + """An exhausted proposer ends successfully with the best evaluated candidate.""" + baseline = Candidate(run_id="run-1", label="agent-0", round=0, optimization="baseline") + tree = EvolutionTree() + tree.add(baseline) + run = ExperimentRun( + workspace="default", + agent="agent", + config_snapshot={}, + status="running", + rounds_completed=0, + ) + run._id = "run-1" + evaluation = EvaluationResult(id="agent-0-validation", aggregate_metrics={"reward": 1.0}) + backend = SimpleNamespace( + client=None, + get_agent_code=AsyncMock(), + persist_evaluation=AsyncMock(), + update_run=AsyncMock(), + persist_result=AsyncMock(), + ) + finalized = AsyncMock(return_value=baseline) + + monkeypatch.setattr( + loop_module, + "EvaluatorFactory", + lambda: SimpleNamespace(build_evaluator=lambda *args, **kwargs: object()), + ) + monkeypatch.setattr( + loop_module, + "DatasetFactory", + lambda: SimpleNamespace(build_dataset=lambda *args, **kwargs: object()), + ) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_init_structure", + lambda self: (tmp_path / "agents", tmp_path / "analysis", tmp_path / "results"), + ) + monkeypatch.setattr(EvolutionaryOptimizer, "_detect_last_round", lambda self: None) + monkeypatch.setattr(EvolutionaryOptimizer, "_create_experiment_run", AsyncMock(return_value=run)) + monkeypatch.setattr(EvolutionaryOptimizer, "_create_baseline_agent", AsyncMock(return_value=baseline)) + monkeypatch.setattr(EvolutionaryOptimizer, "_update_candidate", AsyncMock()) + monkeypatch.setattr( + EvolutionaryOptimizer, "_evaluate_validation_candidates", AsyncMock(return_value={"agent-0": evaluation}) + ) + monkeypatch.setattr(EvolutionaryOptimizer, "_generate_initial_goal_tree", AsyncMock()) + monkeypatch.setattr(EvolutionaryOptimizer, "_select_survivors", AsyncMock(return_value=[baseline])) + monkeypatch.setattr( + EvolutionaryOptimizer, "_evaluate_train_candidates", AsyncMock(return_value={"agent-0": evaluation}) + ) + monkeypatch.setattr(EvolutionaryOptimizer, "_analyze_round", AsyncMock(return_value="analysis")) + monkeypatch.setattr(EvolutionaryOptimizer, "_update_goal_tree", AsyncMock()) + propose = AsyncMock(return_value=[]) + monkeypatch.setattr(EvolutionaryOptimizer, "_propose_improvements", propose) + monkeypatch.setattr(EvolutionaryOptimizer, "_implement_candidates", AsyncMock()) + monkeypatch.setattr(EvolutionaryOptimizer, "_finalize", finalized) + monkeypatch.setattr(loop_module.EvolutionTree, "from_dir", lambda path: tree) + + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + optimizer.config = EvolutionaryOptimizerConfig() + optimizer.shell = SimpleNamespace(close=AsyncMock()) + optimizer.terminator = SimpleNamespace(run=AsyncMock(return_value=TerminationDecision(stop=False))) + optimizer._framework_skills_dirs = [] + deps = SimpleNamespace( + backend=backend, + workspace="default", + config=EvolutionaryOptimizerConfig(), + evaluator_type="harbor", + train_dataset=object(), + validation_dataset=object(), + insight=None, + agent=tmp_path / "agent", + agent_spec=None, + task_template=None, + ) + + result = await optimizer.run(deps) + + assert result.winner is baseline + propose.assert_awaited_once() + finalized.assert_awaited_once() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py new file mode 100644 index 0000000000..3ef1a684c2 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-weakness unit tests for the smoke agent. No Docker, no network. + +Each test pins one documented behaviour of the baseline agent. A failure here +usually means someone "fixed" the agent; see +plugins/nemo-experimentalist/examples/smoke-agent/README.md first. +""" + +from __future__ import annotations + +import importlib.util +import os +import shutil +import sys +from pathlib import Path +from typing import Any + +import pytest +from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import _ignore_agent_copy +from nemo_experimentalist_plugin.profile import load_profile + +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_RECORDS = _EXAMPLE_DIR / "dataset" / "_shared" / "records.json" + + +@pytest.fixture(scope="module") +def agent_module(tmp_path_factory: pytest.TempPathFactory) -> Any: + """Import agent.py by path; it is not an installed package.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ["TRACE_DIR"] = str(tmp_path_factory.mktemp("traces")) + spec = importlib.util.spec_from_file_location("_smoke_agent", _EXAMPLE_DIR / "agent" / "agent.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module + + +def test_working_lookup_succeeds(agent_module: Any) -> None: + """Check that a normal lookup works at baseline.""" + agent = agent_module.ReportAgent() + assert agent.solve("What is the department of Ada Lovelace?") == "dept=research" + + +def test_g1_no_aggregation_capability(agent_module: Any) -> None: + """Check that the baseline cannot add up hours.""" + agent = agent_module.ReportAgent() + answer = agent.solve("What is the total hours for the research department?") + assert answer == agent_module.FALLBACK + + +def test_g2_punctuated_names_fall_through(agent_module: Any) -> None: + """Check that the baseline cannot look up names with punctuation or accents.""" + agent = agent_module.ReportAgent() + for name in ("O'Brien", "Zoë Washington", "Ann-Marie Cruz"): + assert agent.solve(f"What is the department of {name}?") == agent_module.FALLBACK + + +def test_g3_long_instruction_is_clipped(agent_module: Any) -> None: + """Check that a long preamble hides an otherwise valid question.""" + agent = agent_module.ReportAgent() + preamble = "Reporting policy applies to this request. " * 10 + assert len(preamble) > agent_module.MAX_INSTRUCTION_CHARS + question = "What is the department of Grace Hopper?" + assert agent.solve(question) == "dept=research" + assert agent.solve(preamble + question) == agent_module.FALLBACK + + +def test_g4_list_handler_shadows_count(agent_module: Any) -> None: + """Check that a count question wrongly returns a list.""" + agent = agent_module.ReportAgent() + answer = agent.solve("How many people are in the research department?") + assert answer.startswith("names="), "expected the greedy list handler to win" + assert answer != "count=3" + + +def test_g5_missing_record_does_not_degrade(agent_module: Any) -> None: + """Check that an unknown person falls back instead of giving the requested answer.""" + agent = agent_module.ReportAgent() + assert agent.solve("What is the department of Alan Turing?") == agent_module.FALLBACK + + +def test_g5_empty_field_does_not_degrade(agent_module: Any) -> None: + """Check that an empty record field is returned as an empty answer.""" + agent = agent_module.ReportAgent() + assert agent.solve("What is the role of Karl Jung?") == "role=" + + +def test_agent_is_deterministic(agent_module: Any) -> None: + """Check that repeated identical input gives identical output.""" + agent = agent_module.ReportAgent() + question = "What is the department of Ada Lovelace?" + assert len({agent.solve(question) for _ in range(20)}) == 1 + + +def test_agent_declares_no_strategy_methods() -> None: + """Check that the agent does not declare LLM-backed strategy methods.""" + source = (_EXAMPLE_DIR / "agent" / "agent.py").read_text(encoding="utf-8") + assert "@strategy" not in source + assert "CodeActStrategy" not in source + + +def test_spec_forbids_llm_backed_changes() -> None: + """Check that the spec forbids LLM-backed changes.""" + spec = (_EXAMPLE_DIR / "AGENT-SPEC.md").read_text(encoding="utf-8").lower() + for phrase in ("@strategy", "deterministic", "no llm", "offline"): + assert phrase in spec, f"AGENT-SPEC.md must mention {phrase!r}" + + +# The candidate must receive only these implementation files. Keeping this exact +# list makes an added README, config, dataset, or helper visible in review rather +# than silently giving the Coder more context. +_AGENT_SOURCE_DIRNAME = "agent" +_CANDIDATE_SOURCE_FILES = ("agent.py", "harbor_wrapper.py", "main.py") + + +def test_candidate_copy_contains_only_the_declared_agent_source(tmp_path: Path) -> None: + """Check that the Coder receives only the declared agent source files. + + A profile spelling check cannot prove the effective path or the copied file + set. This guards the boundary that previously leaked a live `.env` into + candidate workspaces: only the three implementation files may arrive. + """ + profile = load_profile(_EXAMPLE_DIR / "optimizer.yaml") + source = (profile.profile_dir / profile.agent_source).resolve() + expected_source = (_EXAMPLE_DIR / _AGENT_SOURCE_DIRNAME).resolve() + assert source == expected_source, f"agent_source resolves to {source}, expected {expected_source}" + + copied = tmp_path / "candidate-source" + shutil.copytree(source, copied, ignore=_ignore_agent_copy) + files = tuple(sorted(path.relative_to(copied).as_posix() for path in copied.rglob("*") if path.is_file())) + assert files == _CANDIDATE_SOURCE_FILES, ( + f"candidate source must contain only the declared implementation files; found {files}" + ) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py new file mode 100644 index 0000000000..50ed0625ec --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_assets.py @@ -0,0 +1,340 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the smoke agent's task image, its NOOA pin, and its verifier honest. + +No Docker here on purpose: the image tag is a content hash, so a forgotten +rebuild is a string comparison rather than something only a container run can +reveal. +""" + +from __future__ import annotations + +import functools +import hashlib +import importlib.util +import re +import shutil +import sys +import tomllib +from pathlib import Path +from typing import Any + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_SHARED = _EXAMPLE_DIR / "dataset" / "_shared" +_HASHED = ("Dockerfile", "records.json") +_RENDERED_TASK_TREE_SHA256 = "3fabb557da0cd6f4cda6c713b261e591bf52ed5ad936f2d3ee7bd6b12431099a" + + +def _root_nooa_rev() -> str: + data = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + return data["tool"]["uv"]["sources"]["nooa"]["rev"] + + +def _expected_tag() -> str: + digest = hashlib.sha256() + for name in _HASHED: + digest.update((_SHARED / name).read_bytes()) + return f"smoke-agent-env:sha-{digest.hexdigest()[:12]}" + + +def _template_toml() -> Path: + """Return the canonical task shape.""" + return _EXAMPLE_DIR / "dataset" / "task-template" / "task.toml" + + +@functools.cache +def _renderer() -> Any: + """Import the renderer by path; scripts is not a package.""" + path = _EXAMPLE_DIR / "scripts" / "render_tasks.py" + spec = importlib.util.spec_from_file_location("_smoke_render_tasks", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module + + +def _tree_sha256(root: Path) -> str: + """Hash a task tree's paths and bytes.""" + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file() and item.name != ".gitignore"): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +_EXPECTED_METRIC_KEYS = ("reward", "shape_ok") + + +def test_verifier_emits_exactly_the_two_metric_keys() -> None: + """Check that the verifier emits exactly the two expected metric keys. + + Asserting only that both names appear somewhere would also accept a third + key, or a second write, either of which changes the metric set every trial + reports. Parse what is actually emitted instead. + """ + code = _verifier_code() + writes = [line.strip() for line in code.splitlines() if "reward.json" in line and "printf" in line] + assert len(writes) == 1, f"expected exactly one line writing reward.json, found {len(writes)}: {writes}" + keys = tuple(re.findall(r'"([A-Za-z_][A-Za-z0-9_]*)"\s*:', writes[0])) + assert keys == _EXPECTED_METRIC_KEYS, f"verifier emits {keys}, expected exactly {_EXPECTED_METRIC_KEYS}" + + +def _verifier_code() -> str: + """Return test.sh with comment lines dropped. + + Every guard below is a substring check, and the script documents each choice + in a comment that names the rejected alternative. Checking raw text would + match those comments rather than the code. + """ + lines = (_SHARED / "test.sh").read_text(encoding="utf-8").splitlines() + return "\n".join(line for line in lines if not line.lstrip().startswith("#")) + + +def _shell_options(code: str) -> tuple[set[str], set[str]]: + """Return (short flags, `-o` long options) enabled across every `set` line. + + Parsed rather than string-matched so the contract holds however it is spelled: + ``set -uo pipefail``, ``set -u -o pipefail`` and ``set -euo pipefail`` all + resolve to the same options. + """ + short: set[str] = set() + long: set[str] = set() + for raw in code.splitlines(): + line = raw.strip() + if not line.startswith("set "): + continue + tokens = line.split()[1:] + index = 0 + while index < len(tokens): + token = tokens[index] + index += 1 + if not token.startswith("-") or token.startswith("--"): + continue + for flag in token[1:]: + if flag == "o" and index < len(tokens): + long.add(tokens[index]) + index += 1 + else: + short.add(flag) + return short, long + + +def test_verifier_sets_exactly_the_intended_shell_options() -> None: + """Check that the verifier enables only the intended shell options. + + Checking only that errexit is absent would also pass a verifier with no + `set` options at all. Both of the others earn their place: without nounset an + unset variable expands to empty and a comparison can succeed against nothing, + and without pipefail a failing stage of a pipeline is invisible. + """ + short, long = _shell_options(_verifier_code()) + assert "u" in short, "nounset is off; an unset variable expands to empty and can score a wrong answer" + assert "pipefail" in long, "pipefail is off; a failing stage of a pipeline would go unnoticed" + assert "e" not in short, ( + "errexit is on; aborting before reward.json is written turns a legitimate 0 into a missing metric" + ) + + +def test_verifier_keeps_its_reward_hacking_guards() -> None: + """Check that the verifier keeps its safeguards against reward hacking.""" + code = _verifier_code() + + assert "tr -d" not in code, "tr -d '\\r' deletes every CR, collapsing sum=42 into a passing sum=42" + assert "cmp -s" in code, "whole-file compare; command substitution strips trailing newlines" + assert "refusing to score" in code, "must fail closed when the expected fixture is unreadable" + assert '[ -L "$OUTPUT" ]' in code, ( + "the agent owns /app/artifacts, so without a -L check it can point output.txt at " + "the expected fixture and have the answer key compared against itself" + ) + + +def test_verifier_rejects_a_symlinked_output_before_reading_it() -> None: + """Check that the verifier rejects a symlink before reading the output. + + A `-L` test placed after `-f` never runs: `-f` follows the link, finds a regular + file at the other end, and scores it. + """ + code = _verifier_code() + branches = [line for line in code.splitlines() if line.lstrip().startswith(("if ", "elif "))] + symlink_at = next((i for i, line in enumerate(branches) if '-L "$OUTPUT"' in line), None) + regular_at = next((i for i, line in enumerate(branches) if '-f "$OUTPUT"' in line), None) + assert symlink_at is not None, "verifier must test for a symlinked output" + assert regular_at is not None, "verifier must still test for a regular output file" + assert symlink_at < regular_at, ( + "the -L branch must precede the -f branch, or it is dead code and the symlink-to-answer-key hack scores 1.0" + ) + + +def test_verifier_does_not_echo_answers() -> None: + """Check that the verifier does not print answer values in its log.""" + code = _verifier_code() + for forbidden in ('cat "$EXPECTED_NORM"', 'cat "$ACTUAL_NORM"'): + assert forbidden not in code, f"{forbidden} publishes ground truth to the trial log" + + +def test_dockerfile_nooa_rev_matches_workspace() -> None: + """Check that the task image uses the workspace's NOOA revision.""" + dockerfile_path = _SHARED / "Dockerfile" + if not dockerfile_path.is_file(): + return # Task 4 creates it. + found = re.search(r"labs-OO-Agents\.git@([0-9a-f]{40})", dockerfile_path.read_text(encoding="utf-8")) + assert found is not None, "Dockerfile must pin NOOA to an explicit revision" + assert found.group(1) == _root_nooa_rev() + + +def test_task_image_runs_as_a_non_root_user() -> None: + """Check that model-written task code does not run as root.""" + dockerfile = (_SHARED / "Dockerfile").read_text(encoding="utf-8") + assert "USER smoke-agent" in dockerfile + assert "useradd --uid 10001" in dockerfile + assert "COPY --chown=smoke-agent:smoke-agent records.json" in dockerfile + + +def test_task_template_references_the_current_image() -> None: + """Check that the template uses the current task image.""" + expected = _expected_tag() + actual = tomllib.loads(_template_toml().read_text(encoding="utf-8"))["environment"]["docker_image"] + assert actual == expected, ( + f"task template references {actual}, current content is {expected}. " + "Run scripts/build_image.py after changing the Dockerfile or records.json." + ) + + +def test_task_template_carries_the_current_verifier() -> None: + """Check that the template uses the canonical verifier.""" + canonical = (_SHARED / "test.sh").read_bytes() + actual = (_template_toml().parent / "tests" / "test.sh").read_bytes() + assert actual == canonical, "task template verifier is stale; copy dataset/_shared/test.sh into the template" + + +def test_the_task_template_carries_the_current_records() -> None: + """Check that the task template uses the current records file. + + A trace holds the question and the agent's wrong answer, never the right one, so + Eval Author cannot infer `` from it -- and an unfilled expectation scores + 0 for every agent, making a healthy run read as a failed repair. The records make + the answer derivable. + + A second copy, so it needs a second guard: a drifted one would have Eval Author + compute answers from records the container does not have. + """ + template_records = _EXAMPLE_DIR / "dataset" / "task-template" / "records.json" + if not template_records.is_file(): + return # optional; only insight mode needs it + assert template_records.read_bytes() == (_SHARED / "records.json").read_bytes(), ( + f"{template_records} has drifted from the canonical records; copy dataset/_shared/records.json into the template" + ) + + +def test_task_template_has_an_empty_environment_dir() -> None: + """Check that the template has the Harbor-required environment directory. + + ``TaskModel.is_valid_dir`` returns False when environment/ is absent, and a + dataset whose tasks all fail that check loads with *zero tasks* rather than + raising -- so a missing directory looks like an empty dataset, not an error. + ``[environment].docker_image`` only makes the Dockerfile inside it optional. + + A Dockerfile there would shadow the prebuilt image and reintroduce the + per-task build the content-hash tag exists to avoid. + """ + environment = _template_toml().parent / "environment" + assert environment.is_dir(), "task template has no environment/; Harbor will not see rendered tasks" + contents = {p.name for p in environment.iterdir()} - {".gitkeep"} + assert not contents, f"task template environment must stay empty, found {sorted(contents)}" + + +def test_renderer_reproduces_the_curated_task_tree(tmp_path: Path) -> None: + """Check that the compact manifest renders the exact checked-in fixture. + + The hash covers every rendered path and its bytes, so it moves whenever + ``tasks.json`` or ``task-template/`` changes. That is intended -- an edit to + either should be a deliberate, reviewed act -- but it means the digest has to + be updated by hand, and a stale digest says nothing about *what* differs. + """ + dataset = tmp_path / "dataset" + shutil.copytree(_EXAMPLE_DIR / "dataset", dataset) + rendered = _renderer().render(dataset) + assert len(rendered) == 50 + actual = _tree_sha256(dataset / "groups") + assert actual == _RENDERED_TASK_TREE_SHA256, ( + f"rendered task tree changed: expected {_RENDERED_TASK_TREE_SHA256}, got {actual}. " + "If the manifest or template edit was intended, set _RENDERED_TASK_TREE_SHA256 to the " + "value above after confirming the rendered tasks are correct." + ) + + +@functools.cache +def _builder() -> Any: + """Import scripts/build_all_group.py by path; scripts/ is not a package. + + The exclusion list lives there, so the test reads it rather than repeating + it -- a second copy would drift the moment a group is added or removed. + """ + path = _EXAMPLE_DIR / "scripts" / "build_all_group.py" + spec = importlib.util.spec_from_file_location("_smoke_build_all_group", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_excluded_groups_stay_out_of_the_combined_set(tmp_path: Path) -> None: + """Check that excluded groups are not part of the combined scenario. + + Combining them leaves the full scenario with no reachable pass criterion, so + an accidental re-inclusion has to fail loudly rather than just lower the score. + """ + dataset = tmp_path / "dataset" + shutil.copytree(_EXAMPLE_DIR / "dataset", dataset) + _renderer().render(dataset) + _builder().assemble(dataset) + combined = dataset / "groups" / "_all" + + excluded_keys = {name.split("-")[0] for name in _builder().EXCLUDED_GROUPS} + assert excluded_keys, "the exclusion list should not be empty; see build_all_group.py" + present = {task.parent.name.split("-")[0] for task in combined.rglob("task.toml")} + assert not (present & excluded_keys), ( + f"combined group contains excluded group(s) {sorted(present & excluded_keys)}; run scripts/build_all_group.py" + ) + + +def test_combined_group_matches_its_sources(tmp_path: Path) -> None: + """Check that the combined scenario matches its source task groups. + + Rebuild with scripts/build_all_group.py after changing any group. + """ + dataset = tmp_path / "dataset" + shutil.copytree(_EXAMPLE_DIR / "dataset", dataset) + _renderer().render(dataset) + _builder().assemble(dataset) + groups = dataset / "groups" + combined = groups / "_all" + + builder = _builder() + expected: dict[str, Path] = {} + for group_name in builder.source_groups(dataset): + group = groups / group_name + key = builder.group_key(group_name) + for split in ("train", "validation"): + for task in sorted((group / split).iterdir()): + if (task / "task.toml").is_file(): + expected[f"{split}/{key}-{task.name}"] = task + + actual = {f"{t.parent.parent.name}/{t.parent.name}": t.parent for t in combined.rglob("task.toml")} + assert set(actual) == set(expected), ( + "combined group is stale; run scripts/build_all_group.py. " + f"missing={sorted(set(expected) - set(actual))} unexpected={sorted(set(actual) - set(expected))}" + ) + for rel, src in expected.items(): + for name in ("instruction.md", "task.toml", "tests/expected.txt"): + assert (actual[rel] / name).read_bytes() == (src / name).read_bytes(), ( + f"combined {rel}/{name} differs from its source; run scripts/build_all_group.py" + ) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py new file mode 100644 index 0000000000..2932230031 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_baseline.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Guard the smoke agent's deliberately weak baseline and its record set. + +The fixture only measures anything because the agent ships with known +weaknesses. A well-meaning edit that closes one silently destroys what an +Experimentalist run is asserted against, so the expected baseline is pinned +here. These tests need no Docker and no network. + +See plugins/nemo-experimentalist/examples/smoke-agent/README.md before changing +either the agent or this file. +""" + +from __future__ import annotations + +import functools +import importlib.util +import json +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path +from typing import Any + +import pytest + +_EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "smoke-agent" +_RECORDS = _EXAMPLE_DIR / "dataset" / "_shared" / "records.json" + + +@functools.cache +def _renderer() -> Any: + """Import the task renderer by path; scripts is not a package.""" + path = _EXAMPLE_DIR / "scripts" / "render_tasks.py" + spec = importlib.util.spec_from_file_location("_smoke_render_tasks", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module + + +@pytest.fixture(scope="module") +def rendered_dataset(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Render the compact task manifest once for these deterministic checks.""" + dataset = tmp_path_factory.mktemp("smoke-agent-dataset") / "dataset" + shutil.copytree(_EXAMPLE_DIR / "dataset", dataset) + _renderer().render(dataset) + return dataset + + +def _records() -> list[dict]: + return json.loads(_RECORDS.read_text(encoding="utf-8")) + + +def test_department_totals_are_pinned() -> None: + """Check that department totals match the task expectations.""" + totals: dict[str, int] = {} + for record in _records(): + totals[record["dept"]] = totals.get(record["dept"], 0) + record["hours"] + assert totals == {"research": 29, "ops": 13} + assert sum(totals.values()) == 42 + + +def test_role_scoped_hours_are_pinned() -> None: + """Check that role totals match the task expectations.""" + by_role: dict[str, int] = {} + for record in _records(): + by_role[record["role"]] = by_role.get(record["role"], 0) + record["hours"] + assert by_role["engineer"] == 20 # train + assert by_role["analyst"] == 9 # validation + + +def test_g2_names_carry_no_other_weakness() -> None: + """Check that G2 names do not also trigger another group’s behavior. + + The predicate mirrors the character class the agent's lookup pattern accepts. + `str.isalpha()` is deliberately not used: it is Unicode-aware, so "Zoë" passes + it while the agent's ASCII-only class rejects the name. + """ + tricky = [r for r in _records() if not re.fullmatch(r"[A-Za-z ]+", r["name"])] + assert {r["name"] for r in tricky} == {"O'Brien", "Zoë Washington", "Ann-Marie Cruz"} + for record in tricky: + assert record["role"] != "", ( + f"{record['name']} now carries a second group's weakness — see " + "examples/smoke-agent/README.md before changing this record" + ) + assert isinstance(record["hours"], int), f"{record['name']} would break another group" + + +def test_g5_empty_field_does_not_touch_g1() -> None: + """Check that G5's empty value does not affect G1 tasks.""" + empty = [r for r in _records() if r["role"] == ""] + assert [r["name"] for r in empty] == ["Karl Jung"] + assert all(isinstance(r["hours"], int) for r in _records()), ( + "an empty `hours` would force a G1 fix to absorb G5's robustness" + ) + + +_EXPECTED_BASELINE: dict[tuple[str, str, str], float] = { + # Train shows two *kinds* of filter -- by department and by role -- so a + # general filter mechanism is the obvious fix. Validation holds new instances + # of those same two kinds, which a general fix reaches and a hardcoded one + # does not. This is what makes G1 a repair scenario rather than a + # generalization one; see examples/smoke-agent/README.md. + ("g1-aggregation", "train", "total-hours-research"): 0.0, + ("g1-aggregation", "train", "total-hours-engineers"): 0.0, + ("g1-aggregation", "train", "lookup-ada"): 1.0, + ("g1-aggregation", "validation", "total-hours-ops"): 0.0, + ("g1-aggregation", "validation", "total-hours-analysts"): 0.0, + ("g1-aggregation", "validation", "lookup-grace"): 1.0, + ("g2-name-patterns", "train", "lookup-obrien"): 0.0, + ("g2-name-patterns", "train", "lookup-zoe"): 0.0, + ("g2-name-patterns", "train", "lookup-ada"): 1.0, + ("g2-name-patterns", "validation", "lookup-ann-marie"): 0.0, + ("g2-name-patterns", "validation", "lookup-role-obrien"): 0.0, + ("g2-name-patterns", "validation", "lookup-grace"): 1.0, + ("g3-long-inputs", "train", "preamble-dept"): 0.0, + ("g3-long-inputs", "train", "preamble-role"): 0.0, + ("g3-long-inputs", "train", "plain-dept"): 1.0, + ("g3-long-inputs", "validation", "preamble-long-dept"): 0.0, + ("g3-long-inputs", "validation", "preamble-hours"): 0.0, + ("g3-long-inputs", "validation", "trailing-prose"): 1.0, + ("g4-dispatch-order", "train", "count-research"): 0.0, + ("g4-dispatch-order", "train", "count-ops"): 0.0, + ("g4-dispatch-order", "train", "lookup-ada"): 1.0, + ("g4-dispatch-order", "validation", "count-operators-ops"): 0.0, + ("g4-dispatch-order", "validation", "count-engineers-research"): 0.0, + ("g4-dispatch-order", "validation", "lookup-grace"): 1.0, + ("g5-edge-cases", "train", "missing-person"): 0.0, + ("g5-edge-cases", "train", "empty-role"): 0.0, + ("g5-edge-cases", "train", "lookup-ada"): 1.0, + ("g5-edge-cases", "validation", "missing-person-role"): 0.0, + ("g5-edge-cases", "validation", "missing-person-hours"): 0.0, + ("g5-edge-cases", "validation", "lookup-grace"): 1.0, +} + +GROUPS = ( + "g1-aggregation", + "g2-name-patterns", + "g3-long-inputs", + "g4-dispatch-order", + "g5-edge-cases", +) + + +@functools.cache +def _agent_class() -> Any: + """Import agent.py by path; it is not an installed package.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ.setdefault("TRACE_DIR", tempfile.mkdtemp(prefix="smoke-baseline-traces-")) + spec = importlib.util.spec_from_file_location("_smoke_baseline_agent", _EXAMPLE_DIR / "agent" / "agent.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module.ReportAgent + + +def _normalize(text: str) -> str: + """Mirror tests/test.sh: strip CR at end-of-line only, then trailing newlines. + + Deliberately not text.replace("\\r", "") -- that is `tr -d '\\r'`, which deletes + every carriage return and would let total=29 collapse into a passing + total=29. + """ + return re.sub(r"\r$", "", text, flags=re.MULTILINE).rstrip("\n") + + +def _reward_for(dataset: Path, group: str, split: str, task_id: str) -> float: + """Replay the container verifier in-process for one task.""" + task = dataset / "groups" / group / split / task_id + instruction = (task / "instruction.md").read_text(encoding="utf-8").strip() + expected = (task / "tests" / "expected.txt").read_text(encoding="utf-8") + written = _agent_class()().solve(instruction) + "\n" + return 1.0 if _normalize(written) == _normalize(expected) else 0.0 + + +@pytest.mark.parametrize(("key", "expected"), list(_EXPECTED_BASELINE.items())) +def test_baseline_rewards_are_pinned(rendered_dataset: Path, key: tuple[str, str, str], expected: float) -> None: + """Check that every task has its expected baseline reward.""" + assert _reward_for(rendered_dataset, *key) == expected, ( + f"{key} no longer scores {expected} at baseline. The agent ships with deliberate " + "weaknesses; see plugins/nemo-experimentalist/examples/smoke-agent/README.md before " + "changing agent.py." + ) + + +@pytest.mark.parametrize("group", GROUPS) +@pytest.mark.parametrize("split", ["train", "validation"]) +def test_each_split_keeps_two_failures_and_one_control(rendered_dataset: Path, group: str, split: str) -> None: + """Check that every split has two failures and one control.""" + expected_ids = {task_id for (g, s, task_id) in _EXPECTED_BASELINE if g == group and s == split} + actual_ids = { + path.name for path in (rendered_dataset / "groups" / group / split).iterdir() if (path / "task.toml").is_file() + } + assert actual_ids == expected_ids, ( + f"{group}/{split} drifted: missing={sorted(expected_ids - actual_ids)} " + f"unexpected={sorted(actual_ids - expected_ids)}" + ) + rewards = sorted(_reward_for(rendered_dataset, group, split, task_id) for task_id in expected_ids) + assert rewards == [0.0, 0.0, 1.0], f"{group}/{split} lost its two-failure/one-control shape: {rewards}" + + +@pytest.mark.parametrize( + ("answer", "expected_shape"), + [ + ("dept=research", 1.0), + ("count=3", 1.0), + ("names=Ada Lovelace", 1.0), + ("role=", 1.0), + ("I do not know how to answer that.", 0.0), + ], +) +def test_shape_metric_discriminates(answer: str, expected_shape: float) -> None: + """Check that the shape metric distinguishes an answer from a fallback. + + Mirrors the grep in tests/test.sh. A "did the agent write a file" metric was + rejected here: this agent always writes one, so it would be constant. + """ + actual = 1.0 if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", answer.splitlines()[0]) else 0.0 + assert actual == expected_shape diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_1_loop_e2e.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_1_loop_e2e.py new file mode 100644 index 0000000000..a84c1c849d --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_1_loop_e2e.py @@ -0,0 +1,739 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end coverage for the insight-driven smoke-agent loop.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path + +import pytest +from experimentalist_smoke_test_types import SandboxRunner + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_FIXTURE = _REPO_ROOT / "plugins" / "nemo-experimentalist" / "examples" / "smoke-agent" +_HOST_PLATFORM_URL = "http://localhost:8080" +_WORKSPACE = "smoke-agent" +_NO_PROXY = "localhost,127.0.0.1,::1,gateway.docker.internal,host.docker.internal" +_RECORDS = _FIXTURE / "dataset" / "_shared" / "records.json" +_REPAIR_GROUPS = ("g1-aggregation", "g2-name-patterns", "g3-long-inputs", "g5-edge-cases") +_INSIGHT_EVIDENCE_TASKS = { + "g1-aggregation": ( + "total-hours-engineers", + "total-hours-research", + "total-hours-analysts", + "total-hours-operators", + "total-hours-ops", + ), + "g2-name-patterns": ( + "lookup-obrien", + "lookup-zoe", + "lookup-ann-marie", + "lookup-obrien-hours", + "lookup-ann-marie-role", + ), + "g3-long-inputs": ( + "preamble-dept", + "preamble-role", + "preamble-dept-zoe", + "preamble-role-grace", + "preamble-hours-obrien", + ), + "g5-edge-cases": ( + "empty-role", + "missing-person", + "missing-person-linus", + "missing-person-marie", + "missing-person-katherine", + ), +} +_ROOT_CAUSE_TERMS = { + "g1-aggregation": ("total", "sum", "aggregat", "arithmetic"), + "g2-name-patterns": ("regex", "apostrophe", "hyphen", "unicode", "character"), + "g3-long-inputs": ("truncat", "max_instruction", "240", "preamble", "clip"), + "g5-edge-cases": ("missing", "empty", "exception", "unknown", "lookup"), +} +_MIN_ROOT_CAUSE_HITS = 2 +_TEMPLATE_DIR = _FIXTURE / "dataset" / "task-template" +_PLACEHOLDER = re.compile(r"<[A-Z][A-Z0-9_]*>") +_FILLABLE = ("instruction.md", "task.toml", "tests/expected.txt") +_GRAMMAR = ( + re.compile(r"what is the \w+ of ", re.IGNORECASE), + re.compile(r"how many .* in the \w+ department", re.IGNORECASE), + re.compile(r"what is the total \w+ in the \w+ (?:department|role)", re.IGNORECASE), +) + + +def _render_tasks(dataset: Path) -> None: + """Render the compact task manifest into a disposable dataset copy.""" + path = _FIXTURE / "scripts" / "render_tasks.py" + spec = importlib.util.spec_from_file_location("_smoke_render_tasks", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + module.render(dataset) + + +# The Analyst is tested by the nemo-insights plugin. This test instead supplies +# a reviewed Insight with fresh, real trace ids so it isolates the Mode 1 loop: +# trace recording, Eval Author, and the Experimentalist itself. +_MOCK_INSIGHTS = { + "g1-aggregation": { + "title": "Total-hours questions fall through to the fallback answer", + "description": ( + "When the prompt asks for the total hours for a selected group (for example, by role or " + "department), smoke-agent's ordered handler dispatch runs handle_lookup, handle_list, and " + "handle_count, but none of them match the aggregate-total form. The top-level solve span then " + "returns the fixed fallback string 'I do not know how to answer that.' instead of writing the " + "required single-line total= answer. This diverges from the contract that sums over " + "records must be reported with the canonical total= key, and suggests the deterministic regex " + "handler set is missing a total/sum-hours handler for grouped selections." + ), + }, + "g2-name-patterns": { + "title": "Names containing punctuation or non-ASCII characters do not resolve", + "description": ( + "When the prompt asks for a record whose name contains an apostrophe, hyphen, or non-ASCII " + "character, smoke-agent's handle_lookup method does not resolve that person. The lookup regex " + "only accepts ASCII letters and spaces, so it drops the significant part of names such as " + "O'Brien and Zoë Washington before the records lookup runs. The method then raises while " + "searching for the altered name, and solve returns the fixed fallback string instead of the " + "required canonical field=value answer. This suggests that the name-matching pattern must retain " + "the characters that the records file permits." + ), + }, + "g3-long-inputs": { + "title": "Long instructions lose the actual question before dispatch", + "description": ( + "When a reporting-policy preamble comes before an otherwise valid records question, smoke-agent " + "does not reach the question form its handlers recognise. solve truncates the instruction before " + "dispatch, so the end of a long prompt, including the requested lookup, is removed. None of " + "handle_lookup, handle_list, or handle_count then match, and the method returns the fixed fallback " + "string instead of the required canonical field=value answer. This diverges from the agent's " + "reporting contract and suggests the instruction-length limit must be raised or removed." + ), + }, + "g5-edge-cases": { + "title": "Missing records and empty fields return the fallback answer", + "description": ( + "When the prompt asks for a record that is absent or for a field whose value is empty, " + "smoke-agent's lookup path does not produce the documented graceful value. handle_lookup raises " + "while searching for a missing record or formats an empty value, and solve catches the exception " + "only at the top level. It then returns the fixed fallback string rather than writing the required " + "field=unknown answer. This diverges from the records-report contract and suggests lookup failures " + "and empty fields need explicit, per-field handling." + ), + }, +} + + +@dataclass(frozen=True) +class _ExperimentCase: + """One Mode 1 loop configuration.""" + + group: str + generated_only: bool + + +@dataclass(frozen=True) +class _Experiment: + """One downloaded Mode 1 experiment.""" + + case: _ExperimentCase + path: Path + + +_EXPERIMENT_CONFIGURATIONS = tuple( + _ExperimentCase(group, generated_only) for group in _REPAIR_GROUPS for generated_only in (False, True) +) +_EXPERIMENT_CASES = tuple( + pytest.param( + case, + id=f"{case.group}-{'generated-only' if case.generated_only else 'augmented'}", + marks=pytest.mark.xdist_group( + f"mode-1-{case.group}-{'generated-only' if case.generated_only else 'augmented'}" + ), + ) + for case in _EXPERIMENT_CONFIGURATIONS +) +_GENERATED_ONLY_CASES = tuple( + pytest.param( + case, + id=f"{case.group}-generated-only", + marks=pytest.mark.xdist_group(f"mode-1-{case.group}-generated-only"), + ) + for case in _EXPERIMENT_CONFIGURATIONS + if case.generated_only +) + + +def _require_e2e_environment() -> None: + """Check that the host services required by the handover procedure are available.""" + platform = subprocess.run( + ["curl", "-sf", f"{_HOST_PLATFORM_URL}/health/ready"], + capture_output=True, + text=True, + check=False, + ) + if platform.returncode != 0: + pytest.skip(f"start the Platform on {_HOST_PLATFORM_URL} before running the smoke-agent E2E tests") + + +def _process_environment() -> dict[str, str]: + """Build the minimum environment shared by recording and optimization commands.""" + return { + "NO_PROXY": _NO_PROXY, + "no_proxy": _NO_PROXY, + } + + +def _record_trace_ids( + runtime: SandboxRunner, + *, + group: str, + remote_fixture: str, + workspace: str, + remote_artifact_parent: str, + log: Path, +) -> list[str]: + """Record the group's train traces and return the published failing trace ids.""" + output = runtime.run( + [ + "uv", + "run", + "--frozen", + "--python", + "3.13", + "--package", + "nemo-experimentalist-plugin", + "--with", + "./plugins/nemo-agents", + "python", + f"{remote_fixture}/scripts/record_traces.py", + "--group", + group, + "--split", + "insight-evidence", + "--workspace", + workspace, + "--agent", + f"{remote_fixture}/agent", + "--dataset-root", + f"{remote_fixture}/dataset", + "--output", + f"{remote_artifact_parent}/recordings", + "--base-url", + runtime.platform_url, + ], + log=log, + environment=_process_environment(), + capture_output=True, + ) + published = dict(re.findall(r"^(\S+)\s+([0-9a-f]{32})$", output, flags=re.MULTILINE)) + expected = _INSIGHT_EVIDENCE_TASKS[group] + missing = sorted(set(expected) - set(published)) + assert not missing, f"{group} recording did not publish failing task traces {missing}; see {log}" + trace_ids = [published[task_id] for task_id in expected] + assert len(trace_ids) == len(set(trace_ids)), f"{group} recording published duplicate trace ids: {trace_ids}" + assert len(trace_ids) >= 5, f"{group} Insight has too few evidence traces: {trace_ids}" + return trace_ids + + +def _write_mock_insight( + *, + group: str, + workspace: str, + trace_ids: list[str], + path: Path, +) -> str: + """Write one reviewed Insight that points at this run's recorded traces.""" + template = _MOCK_INSIGHTS[group] + insight_id = f"mock-{group}-{uuid.uuid4().hex}" + payload = { + "insights": [ + { + "id": insight_id, + "workspace": workspace, + "name": f"mock-{group}", + "title": template["title"], + "agent": "smoke-agent", + "description": template["description"], + "status": "open", + "trace_refs": trace_ids, + } + ] + } + path.parent.mkdir(parents=True, exist_ok=True) + # JSON is valid YAML. Keeping this dependency-free makes the fixture's + # mocked Analyst output easy to inspect in a failed pytest directory. + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return insight_id + + +def _run_experimentalist( + runtime: SandboxRunner, + *, + remote_fixture: str, + remote_experiment: str, + remote_insight: str, + insight_id: str, + workspace: str, + log: Path, +) -> None: + """Run the insight-driven loop in the selected boundary with one mocked Insight.""" + command = [ + "uv", + "run", + "--frozen", + "--python", + "3.13", + "--package", + "nemo-experimentalist-plugin", + "--with", + "./plugins/nemo-agents", + "nemo", + "agents", + "experimentalist", + "run", + "--profile", + f"{remote_fixture}/optimizer.yaml", + "--insight", + remote_insight, + "--insight-id", + insight_id, + "--workspace", + workspace, + "--base-url", + runtime.platform_url, + "--config", + f"{remote_fixture}/configs/short.yaml", + "--experiment-dir", + remote_experiment, + ] + runtime.run(command, log=log, environment=_process_environment()) + + +def _winner_label(experiment: Path) -> str: + """Read the selected winner from the completed run.""" + run = json.loads((experiment / "eval-and-optimize" / "run.json").read_text(encoding="utf-8")) + winner = run.get("winner_agent") + assert winner, f"run.json has no winner_agent: {sorted(run)}" + return str(winner) + + +def _agent_source(experiment: Path, label: str) -> str: + """Read the saved source for one candidate.""" + return (experiment / "eval-and-optimize" / "agents" / label / "agent.py").read_text(encoding="utf-8") + + +def _replays_correctly(experiment: Path, label: str, task_dir: Path) -> bool: + """Check one committed task against a saved candidate.""" + return _agent_replays_correctly(experiment / "eval-and-optimize" / "agents" / label / "agent.py", label, task_dir) + + +def _agent_replays_correctly(agent_path: Path, label: str, task_dir: Path) -> bool: + """Check one task against an agent source file.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ.setdefault("TRACE_DIR", tempfile.mkdtemp(prefix="smoke-mode-1-traces-")) + spec = importlib.util.spec_from_file_location(f"_smoke_mode_1_{label}", agent_path) + assert spec is not None and spec.loader is not None, f"cannot import {agent_path}" + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + + instruction = (task_dir / "instruction.md").read_text(encoding="utf-8").strip() + expected = (task_dir / "tests" / "expected.txt").read_text(encoding="utf-8") + actual = module.ReportAgent().solve(instruction) + "\n" + return _normalize(actual) == _normalize(expected) + + +@pytest.mark.parametrize("group", _REPAIR_GROUPS) +def test_insight_evidence_tasks_fail_on_the_baseline(group: str, tmp_path: Path) -> None: + """Check that five Insight evidence tasks show the group's baseline failure.""" + local_fixture = tmp_path / "smoke-agent" + shutil.copytree(_FIXTURE, local_fixture) + _render_tasks(local_fixture / "dataset") + evidence = local_fixture / "dataset" / "groups" / group / "insight-evidence" + tasks = [evidence / name for name in _INSIGHT_EVIDENCE_TASKS[group]] + assert all(task.is_dir() for task in tasks), f"{group} did not create all Insight evidence tasks" + still_passing = [ + task.name + for task in tasks + if _agent_replays_correctly(local_fixture / "agent" / "agent.py", f"baseline_{task.name}", task) + ] + assert not still_passing, f"{group} Insight evidence does not show the baseline failure: {still_passing}" + + +def _normalize(text: str) -> str: + """Normalize line endings the same way the task verifier does.""" + return re.sub(r"\r$", "", text, flags=re.MULTILINE).rstrip("\n") + + +def _template_placeholders() -> set[str]: + """Read every placeholder the committed task template declares.""" + found: set[str] = set() + for name in _FILLABLE: + path = _TEMPLATE_DIR / name + if path.is_file(): + found.update(_PLACEHOLDER.findall(path.read_text(encoding="utf-8"))) + return found + + +def _suite_dirs(experiment: Path) -> list[Path]: + """Find the materialized Insight suites from their manifests.""" + root = experiment / "eval-and-optimize" / "eval_author" + return sorted(manifest.parent for manifest in root.rglob("insight-suite/manifest.json")) if root.is_dir() else [] + + +def _materialized_tasks(suite: Path) -> list[Path]: + """Read only the task directories listed in one suite manifest.""" + tasks = json.loads((suite / "manifest.json").read_text(encoding="utf-8")).get("tasks") + return ( + [suite / entry["path"] for entry in tasks if isinstance(entry, dict) and entry.get("path")] + if isinstance(tasks, list) + else [] + ) + + +def _authored_metric_keys(experiment: Path) -> set[str]: + """Read the metrics Eval Author declared in its generated tasks.""" + keys: set[str] = set() + for suite in _suite_dirs(experiment): + for task in _materialized_tasks(suite): + contract = task / "tests" / "metric-contract.json" + if contract.is_file(): + keys.update(json.loads(contract.read_text(encoding="utf-8")).get("metric_keys") or []) + return keys + + +def _baseline_split_metrics(experiment: Path, split: str) -> dict[str, float]: + """Read the baseline aggregate metrics for one evaluated split.""" + result = experiment / "eval-and-optimize" / "results" / f"agent-0-{split}" / "result.json" + evaluations = list(json.loads(result.read_text(encoding="utf-8"))["stats"]["evals"].values()) + assert len(evaluations) == 1, f"expected one evaluation in {result}, found {len(evaluations)}" + metrics = evaluations[0]["metrics"] + assert len(metrics) == 1, f"expected one aggregate metric record in {result}, found {len(metrics)}" + return {str(name): float(value) for name, value in metrics[0].items()} + + +def _generated_trial_rewards(experiment: Path) -> list[dict[str, float]]: + """Read baseline verifier rewards for the tasks Eval Author generated.""" + generated: list[dict[str, float]] = [] + for result in sorted((experiment / "eval-and-optimize" / "results").glob("agent-0-*/*/result.json")): + payload = json.loads(result.read_text(encoding="utf-8")) + if not str(payload.get("task_name", "")).startswith("smoke/generated__"): + continue + rewards = payload.get("verifier_result", {}).get("rewards") + assert isinstance(rewards, dict), f"generated task {result.parent.name} has no verifier rewards" + generated.append({str(name): float(value) for name, value in rewards.items()}) + assert generated, "no generated task produced a baseline trial result" + return generated + + +def _check_insight_suite(experiment: Path, dataset: Path) -> None: + """Check that Mode 1 produced usable generated tasks and objective metrics.""" + declared = _template_placeholders() + assert declared, f"{_TEMPLATE_DIR} declares no tokens" + unmatched_curated = [ + path.parent.name + for path in sorted((dataset / "groups").rglob("instruction.md")) + if not any(pattern.search(path.read_text(encoding="utf-8")) for pattern in _GRAMMAR) + ] + assert not unmatched_curated, "curated tasks fall outside the agent grammar: " + ", ".join(unmatched_curated) + suites = _suite_dirs(experiment) + assert suites, "Eval Author did not materialize an Insight suite" + tasks = [task for suite in suites for task in _materialized_tasks(suite)] + assert tasks, "Insight suite manifests list no tasks" + + unfilled: list[str] = [] + empty_expected: list[str] = [] + off_grammar: list[str] = [] + for task in tasks: + for name in _FILLABLE: + path = task / name + if path.is_file() and ( + remaining := sorted(set(_PLACEHOLDER.findall(path.read_text(encoding="utf-8"))) & declared) + ): + unfilled.append(f"{task.name}/{name}: {', '.join(remaining)}") + expected = task / "tests" / "expected.txt" + if expected.is_file() and not expected.read_text(encoding="utf-8").strip(): + empty_expected.append(task.name) + instruction = task / "instruction.md" + if instruction.is_file() and not any( + pattern.search(instruction.read_text(encoding="utf-8")) for pattern in _GRAMMAR + ): + off_grammar.append(task.name) + assert not unfilled, "Eval Author left template placeholders: " + "; ".join(unfilled) + assert not empty_expected, "generated tasks have empty expected answers: " + ", ".join(empty_expected) + assert not off_grammar, "generated questions fall outside the agent grammar: " + ", ".join(off_grammar) + + authored = _authored_metric_keys(experiment) + assert authored, "Eval Author wrote no metric contract for the generated suite" + missing_from_trials = [ + sorted(authored - rewards.keys()) + for rewards in _generated_trial_rewards(experiment) + if not authored <= rewards.keys() + ] + assert not missing_from_trials, "generated task verifier results dropped authored metric keys: " + "; ".join( + ", ".join(keys) for keys in missing_from_trials + ) + for split in ("train", "validation"): + missing = sorted(authored - _baseline_split_metrics(experiment, split).keys()) + assert not missing, f"baseline {split} aggregate dropped authored metric keys: {missing}" + + run = json.loads((experiment / "eval-and-optimize" / "run.json").read_text(encoding="utf-8")) + objectives = run.get("config_snapshot", {}).get("objective_function") + assert isinstance(objectives, list), "run.json has no objective_function in config_snapshot" + objective_names = {str(metric.get("name")) for metric in objectives if isinstance(metric, dict)} + assert objective_names == authored, ( + f"Mode 1 objectives {sorted(objective_names)} do not match Eval Author metrics {sorted(authored)}" + ) + + results = experiment / "eval-and-optimize" / "results" + for task in tasks: + matches = [ + trial + for trial in results.rglob("*") + if trial.is_dir() + and (base := re.sub(r"__[A-Za-z0-9]+$", "", trial.name)) + and len(base) >= 12 + and task.name.startswith(base) + ] + assert matches and any((trial / "verifier" / "reward.json").is_file() for trial in matches), ( + f"generated task {task.name} produced no reward.json" + ) + + +def _assert_committed_validation_passes(experiment: Path, group: str, dataset: Path) -> None: + """Check that the Mode 1 winner repairs every committed validation task.""" + winner = _winner_label(experiment) + assert winner != "agent-0", f"{group} retained the baseline; nothing was repaired" + assert _agent_source(experiment, winner) != _agent_source(experiment, "agent-0"), ( + f"{group} winner source is identical to the baseline" + ) + validation = dataset / "groups" / group / "validation" + tasks = [task for task in sorted(validation.iterdir()) if (task / "task.toml").is_file()] + assert tasks, f"no validation tasks under {validation}; the fixture moved" + failed = [task.name for task in tasks if not _replays_correctly(experiment, winner, task)] + assert not failed, f"{winner} does not answer held-out {group} tasks: {failed}" + + +def _assert_loop_only_evaluated_generated_tasks(experiment: Path) -> None: + """Check that the Mode 1 loop evaluated only tasks created from the Insight.""" + results = experiment / "eval-and-optimize" / "results" + task_names: set[str] = set() + for result in sorted(results.glob("agent-*/*/result.json")): + payload = json.loads(result.read_text(encoding="utf-8")) + task_name = payload.get("task_name") + if isinstance(task_name, str): + task_names.add(task_name) + assert task_names, "the loop produced no trial results" + leaked = sorted(name for name in task_names if not name.startswith("smoke/generated__")) + assert not leaked, f"the Mode 1 loop evaluated committed tasks instead of only the Insight suite: {leaked}" + + +def _validation_metrics(experiment: Path, label: str) -> dict[str, float]: + """Read one candidate's final validation metric values.""" + path = experiment / "eval-and-optimize" / "agents" / label / "metadata.json" + metrics = (json.loads(path.read_text(encoding="utf-8")).get("rewards", {}).get("validation") or {}).get( + "metrics", {} + ) + assert isinstance(metrics, dict), f"{label} has no validation metrics" + return {str(name): float(value) for name, value in metrics.items()} + + +def _assert_winner_improves_objectives_without_regression(experiment: Path) -> None: + """Check that the selected Mode 1 winner improves objectives and preserves guardrails.""" + run = json.loads((experiment / "eval-and-optimize" / "run.json").read_text(encoding="utf-8")) + winner = _winner_label(experiment) + assert winner != "agent-0", "Mode 1 retained the baseline instead of selecting an improved winner" + baseline = _validation_metrics(experiment, "agent-0") + selected = _validation_metrics(experiment, winner) + snapshot = run.get("config_snapshot", {}) + objectives = snapshot.get("objective_function") + regressions = snapshot.get("regression_metrics") + assert isinstance(objectives, list) and objectives, "run.json has no Mode 1 objective metrics" + assert isinstance(regressions, list), "run.json has no Mode 1 regression metrics" + + for target in objectives: + assert isinstance(target, dict) and isinstance(target.get("name"), str), f"invalid objective target: {target}" + name = target["name"] + assert name in baseline and name in selected, f"objective {name!r} is missing from winner or baseline metrics" + assert selected[name] > baseline[name], ( + f"winner {winner} did not improve objective {name!r}: {baseline[name]} -> {selected[name]}" + ) + + for target in regressions: + assert isinstance(target, dict) and isinstance(target.get("name"), str), f"invalid regression target: {target}" + name = target["name"] + direction = target.get("direction") + assert direction in {"maximize", "minimize"}, f"invalid regression direction for {name!r}: {direction!r}" + assert name in baseline and name in selected, f"regression {name!r} is missing from winner or baseline metrics" + worsened = selected[name] < baseline[name] if direction == "maximize" else selected[name] > baseline[name] + assert not worsened, f"winner {winner} regressed {name!r}: {baseline[name]} -> {selected[name]}" + + +def _assert_analysis_named_problem(experiment: Path, group: str) -> None: + """Check that the Analyzer named the problem measured by this group.""" + analyses = sorted((experiment / "eval-and-optimize" / "analysis").glob("round-*.md")) + assert analyses, f"{group} has no Analyzer output" + text = " ".join(path.read_text(encoding="utf-8") for path in analyses).lower() + hits = [term for term in _ROOT_CAUSE_TERMS[group] if term in text] + assert len(hits) >= _MIN_ROOT_CAUSE_HITS, f"{group} analysis did not name its problem; matched only {hits}" + + +def _run_mode_1_case( + group: str, + tmp_path: Path, + runtime: SandboxRunner, + *, + generated_only: bool, +) -> Path: + """Run and download one Mode 1 group with either augmented or generated-only data.""" + _require_e2e_environment() + mode = "generated-only" if generated_only else "augmented" + artifact_parent = tmp_path / f"{group}-{mode}" + experiment = artifact_parent / "experiment" + log = artifact_parent / "run.log" + workspace = f"smoke-agent-e2e-{group}-{uuid.uuid4().hex[:8]}" + artifact_parent.mkdir(parents=True) + remote_fixture, remote_experiment = runtime.prepare_fixture(artifact_parent, log=log) + remote_artifact_parent = str(Path(remote_experiment).parent) + profile = f"{remote_fixture}/optimizer.yaml" + runtime.replace_text(profile, "g1-aggregation", group, log=log) + runtime.replace_text(profile, "workspace: default", f"workspace: {workspace}", log=log) + if generated_only: + runtime.make_directories( + f"{remote_fixture}/generated-only/train", + f"{remote_fixture}/generated-only/validation", + log=log, + ) + runtime.replace_text(profile, f"./dataset/groups/{group}/train", "./generated-only/train", log=log) + runtime.replace_text(profile, f"./dataset/groups/{group}/validation", "./generated-only/validation", log=log) + if group == "g5-edge-cases": + runtime.replace_text( + f"{remote_fixture}/configs/short.yaml", + "disable_trajectory_scoring: true", + "disable_trajectory_scoring: false", + log=log, + ) + + if os.environ.get("SMOKE_AGENT_IMAGE_BUILT") != "1": + runtime.run( + ["uv", "run", "--no-project", f"{remote_fixture}/scripts/build_image.py"], + log=log, + ) + trace_ids = _record_trace_ids( + runtime, + group=group, + remote_fixture=remote_fixture, + workspace=workspace, + remote_artifact_parent=remote_artifact_parent, + log=log, + ) + insight = artifact_parent / "insights" / f"{group}.yaml" + insight_id = _write_mock_insight( + group=group, + workspace=workspace, + trace_ids=trace_ids, + path=insight, + ) + remote_insights = f"{remote_artifact_parent}/insights" + runtime.make_directories(remote_insights, log=log) + runtime.copy_in(insight, remote_insights, log=log) + _run_experimentalist( + runtime, + remote_fixture=remote_fixture, + remote_experiment=remote_experiment, + remote_insight=f"{remote_insights}/{insight.name}", + insight_id=insight_id, + workspace=workspace, + log=log, + ) + runtime.fetch(remote_experiment, artifact_parent, log=log) + assert experiment.is_dir(), f"Experimentalist did not create the experiment directory at {experiment}" + return experiment + + +@pytest.fixture(scope="session") +def experiment( + request: pytest.FixtureRequest, + sandbox_runner: SandboxRunner, + tmp_path_factory: pytest.TempPathFactory, +) -> _Experiment: + """Run and download one Mode 1 case.""" + case = request.param + assert isinstance(case, _ExperimentCase) + path = _run_mode_1_case( + case.group, + tmp_path_factory.mktemp(f"mode-1-{case.group}"), + sandbox_runner, + generated_only=case.generated_only, + ) + return _Experiment(case, path) + + +@pytest.fixture(scope="session") +def rendered_dataset(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Render curated tasks for host-side candidate checks.""" + dataset = tmp_path_factory.mktemp("smoke-agent-dataset") / "dataset" + shutil.copytree(_FIXTURE / "dataset", dataset) + _render_tasks(dataset) + return dataset + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _EXPERIMENT_CASES, indirect=True) +def test_insight_driven_loop_materializes_a_usable_suite(experiment: _Experiment, rendered_dataset: Path) -> None: + """Check that each downloaded Mode 1 experiment has usable generated tasks and objectives.""" + _check_insight_suite(experiment.path, rendered_dataset) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _EXPERIMENT_CASES, indirect=True) +def test_insight_driven_loop_improves_objectives(experiment: _Experiment) -> None: + """Check that each downloaded Mode 1 winner improves objectives without regression.""" + _assert_winner_improves_objectives_without_regression(experiment.path) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _GENERATED_ONLY_CASES, indirect=True) +def test_generated_only_mode_1_uses_only_generated_tasks(experiment: _Experiment) -> None: + """Check that generated-only Mode 1 experiments never evaluate committed tasks in the loop.""" + _assert_loop_only_evaluated_generated_tasks(experiment.path) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _GENERATED_ONLY_CASES, indirect=True) +def test_generated_only_mode_1_repairs_committed_holdout(experiment: _Experiment, rendered_dataset: Path) -> None: + """Check that generated-only Mode 1 winners repair their untouched committed validation tasks.""" + _assert_committed_validation_passes(experiment.path, experiment.case.group, rendered_dataset) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _EXPERIMENT_CASES, indirect=True) +def test_mode_1_analysis_names_the_problem(experiment: _Experiment) -> None: + """Check that each downloaded Mode 1 experiment contains the expected diagnosis.""" + _assert_analysis_named_problem(experiment.path, experiment.case.group) diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_2_loop_e2e.py b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_2_loop_e2e.py new file mode 100644 index 0000000000..b5acc4774e --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_smoke_agent_mode_2_loop_e2e.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end coverage for the dataset-driven smoke-agent loop.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +from experimentalist_smoke_test_types import SandboxRunner + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_FIXTURE = _REPO_ROOT / "plugins" / "nemo-experimentalist" / "examples" / "smoke-agent" +_NO_PROXY = "localhost,127.0.0.1,::1,gateway.docker.internal,host.docker.internal" +_RECORDS = _FIXTURE / "dataset" / "_shared" / "records.json" +_G4_CONTROL_TASKS = {"smoke/g4-lookup-grace"} +_REWARD_DELTA_THRESHOLD = 0.3 +_ROOT_CAUSE_TERMS = { + "g1-aggregation": ("total", "sum", "aggregat", "arithmetic"), + "g2-name-patterns": ("regex", "apostrophe", "hyphen", "unicode", "character"), + "g3-long-inputs": ("truncat", "max_instruction", "240", "preamble", "clip"), + "g5-edge-cases": ("missing", "empty", "exception", "unknown", "lookup"), +} +_MIN_ROOT_CAUSE_HITS = 2 + + +@dataclass(frozen=True) +class _ExperimentCase: + """One Mode 2 loop configuration.""" + + group: str + profile: str = "optimizer.yaml" + + +@dataclass(frozen=True) +class _Experiment: + """One downloaded Mode 2 experiment.""" + + case: _ExperimentCase + path: Path + + +_REPAIR_CASES = tuple( + pytest.param(_ExperimentCase(group), id=group, marks=pytest.mark.xdist_group(f"mode-2-{group}")) + for group in ("g1-aggregation", "g2-name-patterns", "g3-long-inputs", "g5-edge-cases") +) +_G4_CASE = pytest.param( + _ExperimentCase("g4-dispatch-order", "optimizer-generalization.yaml"), + id="g4-dispatch-order", + marks=pytest.mark.xdist_group("mode-2-g4-dispatch-order"), +) + + +def _require_e2e_environment() -> None: + """Check that the host services required by the handover procedure are available.""" + platform = subprocess.run( + ["curl", "-sf", "http://localhost:8080/health/ready"], + capture_output=True, + text=True, + check=False, + ) + if platform.returncode != 0: + pytest.skip("start the Platform on http://localhost:8080 before running the smoke-agent E2E tests") + + +@pytest.fixture(scope="session") +def _e2e_environment(tmp_path_factory: pytest.TempPathFactory, sandbox_runner: SandboxRunner) -> None: + """Prepare the shared sandbox environment used by every E2E group.""" + _require_e2e_environment() + if os.environ.get("SMOKE_AGENT_IMAGE_BUILT") != "1": + log = tmp_path_factory.mktemp("smoke-agent-e2e") / "host.log" + sandbox_runner.run( + [ + "uv", + "run", + "--no-project", + sandbox_runner.source_path(_FIXTURE / "scripts" / "build_image.py"), + ], + log=log, + ) + + +def _run_e2e_command(runtime: SandboxRunner, command: list[str], *, log: Path) -> None: + """Run one Experimentalist command inside the selected isolation boundary.""" + process_environment = {"NO_PROXY": _NO_PROXY, "no_proxy": _NO_PROXY} + runtime.run(command, log=log, environment=process_environment) + + +def _run_group( + group: str, + *, + runtime: SandboxRunner, + artifact_parent: Path, + profile_name: str = "optimizer.yaml", +) -> tuple[Path, Path]: + """Run one group from a separate sandbox-side copy of the smoke fixture.""" + artifact_parent.mkdir(parents=True, exist_ok=True) + log = artifact_parent / "run.log" + local_fixture, remote_experiment = runtime.prepare_fixture(artifact_parent, log=log) + experiment = artifact_parent / "experiment" + profile = f"{local_fixture}/{profile_name}" + config = f"{local_fixture}/configs/short.yaml" + if group != "g1-aggregation": + runtime.replace_text(profile, "g1-aggregation", group, log=log) + if group == "g5-edge-cases": + runtime.replace_text(config, "disable_trajectory_scoring: true", "disable_trajectory_scoring: false", log=log) + _run_e2e_command( + runtime, + [ + "uv", + "run", + "--frozen", + "--python", + "3.13", + "--package", + "nemo-experimentalist-plugin", + "--with", + "./plugins/nemo-agents", + "nemo", + "agents", + "experimentalist", + "run", + "--profile", + profile, + "--no-insight", + "--base-url", + runtime.platform_url, + "--config", + config, + "--experiment-dir", + remote_experiment, + ], + log=log, + ) + runtime.fetch(remote_experiment, artifact_parent, log=log) + assert experiment.is_dir(), f"Experimentalist did not create the experiment directory at {experiment}" + return experiment, log + + +def _aggregate_metrics(experiment: Path, label: str, dataset: str = "validation") -> dict[str, float]: + """Read one candidate's aggregate metrics from its Harbor result.""" + result = experiment / "eval-and-optimize" / "results" / f"{label}-{dataset}" / "result.json" + evaluations = list(json.loads(result.read_text(encoding="utf-8"))["stats"]["evals"].values()) + assert len(evaluations) == 1, f"expected one evaluation in {result}, found {len(evaluations)}" + return evaluations[0]["metrics"][0] + + +def _validation_reward(experiment: Path, label: str) -> float: + """Read one candidate's validation reward from its Harbor result.""" + return float(_aggregate_metrics(experiment, label)["reward"]) + + +def _per_task_rewards(experiment: Path, label: str, dataset: str = "validation") -> dict[str, float]: + """Read each task reward from one candidate's Harbor result.""" + result_dir = experiment / "eval-and-optimize" / "results" / f"{label}-{dataset}" + rewards: dict[str, float] = {} + for trial in sorted(result_dir.glob("*/result.json")): + payload = json.loads(trial.read_text(encoding="utf-8")) + rewards[payload["task_name"]] = float(payload["verifier_result"]["rewards"]["reward"]) + return rewards + + +def _agent_source(experiment: Path, label: str) -> str: + """Read the saved source for one candidate.""" + return (experiment / "eval-and-optimize" / "agents" / label / "agent.py").read_text(encoding="utf-8") + + +def _agent_class(experiment: Path, label: str) -> Any: + """Load one candidate's agent class from its saved source file.""" + os.environ["RECORDS_PATH"] = str(_RECORDS) + os.environ.setdefault("TRACE_DIR", tempfile.mkdtemp(prefix="smoke-e2e-traces-")) + path = experiment / "eval-and-optimize" / "agents" / label / "agent.py" + spec = importlib.util.spec_from_file_location(f"_smoke_e2e_{label}", path) + assert spec is not None and spec.loader is not None, f"cannot import {path}" + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + return module.ReportAgent + + +def _normalize(text: str) -> str: + """Normalize line endings the same way the task verifier does.""" + return re.sub(r"\r$", "", text, flags=re.MULTILINE).rstrip("\n") + + +def _replays_correctly(experiment: Path, label: str, task_dir: Path) -> bool: + """Check one rendered task against a saved candidate.""" + instruction = (task_dir / "instruction.md").read_text(encoding="utf-8").strip() + expected = (task_dir / "tests" / "expected.txt").read_text(encoding="utf-8") + actual = _agent_class(experiment, label)().solve(instruction) + "\n" + return _normalize(actual) == _normalize(expected) + + +def _winner_label(experiment: Path) -> str: + """Read the selected winner from the completed run.""" + run = json.loads((experiment / "eval-and-optimize" / "run.json").read_text(encoding="utf-8")) + winner = run.get("winner_agent") + assert winner, f"run.json has no winner_agent: {sorted(run)}" + return str(winner) + + +def _assert_repair_group(experiment: Path, group: str, dataset: Path) -> None: + """Check that one repair group changed source and passes every held-out task.""" + winner = _winner_label(experiment) + assert winner != "agent-0", f"{group} retained the baseline; nothing was repaired" + assert _agent_source(experiment, winner) != _agent_source(experiment, "agent-0"), ( + f"{group} winner source is identical to the baseline" + ) + validation = dataset / "groups" / group / "validation" + tasks = [task for task in sorted(validation.iterdir()) if (task / "task.toml").is_file()] + assert tasks, f"no validation tasks under {validation}; the fixture moved" + failed = [task.name for task in tasks if not _replays_correctly(experiment, winner, task)] + assert not failed, f"{winner} does not answer held-out {group} tasks: {failed}" + + baseline = _validation_reward(experiment, "agent-0") + improved = _validation_reward(experiment, winner) + assert improved - baseline >= _REWARD_DELTA_THRESHOLD, ( + f"{group} validation reward {baseline} -> {improved} is below {_REWARD_DELTA_THRESHOLD}" + ) + + +def _assert_analysis_named_problem(experiment: Path, group: str) -> None: + """Check that the Analyzer named the weakness measured by one repair group.""" + analyses = sorted((experiment / "eval-and-optimize" / "analysis").glob("round-*.md")) + assert analyses, f"{group} has no Analyzer output" + text = " ".join(path.read_text(encoding="utf-8") for path in analyses).lower() + hits = [term for term in _ROOT_CAUSE_TERMS[group] if term in text] + assert len(hits) >= _MIN_ROOT_CAUSE_HITS, f"{group} analysis did not name its weakness; matched only {hits}" + + +def _best_train_metrics(experiment: Path, label: str) -> dict[str, float]: + """Read the best train aggregate for one candidate.""" + result_dirs = sorted((experiment / "eval-and-optimize" / "results").glob(f"{label}-train*")) + assert result_dirs, f"no train results for {label}" + metrics: list[dict[str, float]] = [] + for result_dir in result_dirs: + evaluations = list( + json.loads((result_dir / "result.json").read_text(encoding="utf-8"))["stats"]["evals"].values() + ) + assert len(evaluations) == 1, f"expected one evaluation in {result_dir}" + metrics.append(evaluations[0]["metrics"][0]) + return max(metrics, key=lambda values: values["reward"]) + + +def _assert_g4_rejected_narrow_fix(experiment: Path) -> None: + """Check that g4 rejects a train-only fix and keeps the baseline.""" + winner = _winner_label(experiment) + assert winner == "agent-0", f"g4 selected {winner}; validation did not reject the narrow fix" + candidates = sorted( + path.name for path in (experiment / "eval-and-optimize" / "agents").iterdir() if path.name != "agent-0" + ) + assert candidates, "g4 produced no candidates" + baseline_train = _best_train_metrics(experiment, "agent-0")["reward"] + best_candidate = max(candidates, key=lambda label: _best_train_metrics(experiment, label)["reward"]) + best_train = _best_train_metrics(experiment, best_candidate)["reward"] + assert best_train > baseline_train, f"g4 never improved on train ({baseline_train} -> {best_train})" + assert _validation_reward(experiment, best_candidate) <= _validation_reward(experiment, "agent-0"), ( + "g4 train winner also improved validation, so the held-out split did not reject it" + ) + rewards = _per_task_rewards(experiment, "agent-0") + broken_controls = {task for task in _G4_CONTROL_TASKS if rewards.get(task, 0.0) < 1.0} + assert not broken_controls, f"g4 baseline controls are failing: {sorted(broken_controls)}" + + +@pytest.fixture(scope="session") +def experiment( + request: pytest.FixtureRequest, + _e2e_environment: None, + sandbox_runner: SandboxRunner, + tmp_path_factory: pytest.TempPathFactory, +) -> _Experiment: + """Run and download one Mode 2 case.""" + case = request.param + assert isinstance(case, _ExperimentCase) + artifact_parent = tmp_path_factory.mktemp(f"mode-2-{case.group}") + path, _ = _run_group( + case.group, + runtime=sandbox_runner, + artifact_parent=artifact_parent, + profile_name=case.profile, + ) + return _Experiment(case, path) + + +@pytest.fixture(scope="session") +def rendered_dataset(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Render curated tasks for host-side candidate checks.""" + dataset = tmp_path_factory.mktemp("smoke-agent-dataset") / "dataset" + shutil.copytree(_FIXTURE / "dataset", dataset) + path = _FIXTURE / "scripts" / "render_tasks.py" + spec = importlib.util.spec_from_file_location("_smoke_render_tasks", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(spec.name, None) + module.render(dataset) + return dataset + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _REPAIR_CASES, indirect=True) +def test_repair_groups_improve_validation(experiment: _Experiment, rendered_dataset: Path) -> None: + """Check that every repair group improves validation from its downloaded experiment.""" + _assert_repair_group(experiment.path, experiment.case.group, rendered_dataset) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", _REPAIR_CASES, indirect=True) +def test_repair_group_analysis_names_the_problem(experiment: _Experiment) -> None: + """Check that every downloaded repair experiment contains the expected diagnosis.""" + _assert_analysis_named_problem(experiment.path, experiment.case.group) + + +@pytest.mark.e2e +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("experiment", (_G4_CASE,), indirect=True) +def test_g4_rejects_a_non_generalizing_fix(experiment: _Experiment) -> None: + """Check that g4 retains the baseline after validation rejects a narrow fix.""" + _assert_g4_rejected_narrow_fix(experiment.path) diff --git a/plugins/nemo-experimentalist/tests/experimentalist_smoke_test_types.py b/plugins/nemo-experimentalist/tests/experimentalist_smoke_test_types.py new file mode 100644 index 0000000000..f5538a5299 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist_smoke_test_types.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Type contract for the smoke-agent sandbox fixture.""" + +from pathlib import Path +from typing import Protocol + + +class SandboxRunner(Protocol): + """Operations the smoke-agent E2E tests require from their fixture.""" + + @property + def platform_url(self) -> str: ... + + def run( + self, + command: list[str], + *, + log: Path, + environment: dict[str, str] | None = None, + capture_output: bool = False, + ) -> str: ... + + def prepare_fixture(self, artifact_parent: Path, *, log: Path) -> tuple[str, str]: ... + + def source_path(self, path: Path) -> str: ... + + def replace_text(self, path: str, old: str, new: str, *, log: Path) -> None: ... + + def make_directories(self, *paths: str, log: Path) -> None: ... + + def copy_in(self, source: Path, destination: str, *, log: Path) -> None: ... + + def fetch(self, remote_path: str, local_parent: Path, *, log: Path) -> None: ...