feat(experimentalist): add a fast, diagnostic smoke agent fixture

.copyrightignore

@@ -3,6 +3,10 @@ web/vendor/
 release/
 .agents/skills/
 e2e/
+plugins/nemo-experimentalist/examples/smoke-agent/dataset/_shared/records.json
+plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/instruction.md
+plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/records.json
+plugins/nemo-experimentalist/examples/smoke-agent/dataset/task-template/tests/expected.txt
 packages/garak_api/garakapi/_config.py
 packages/garak_api/garakapi/_plugins.py
 packages/garak_api/garakapi/exception.py

plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py

@@ -23,7 +23,6 @@
     Task,
     TrialResult,
 )
-from nemo_experimentalist_plugin.experimentalist.components import cache
 from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools
 from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import (
     Diagnostic,
@@ -486,7 +485,6 @@ async def _run(
                     reporter.note(f"trace analysis failed for {ref}: {result}")
                 analysis_statuses[task.id] = ("failed", str(result))
                 continue
-            cache.store(self.experiment_dir, cache.task_hash(f"eval_author:{ref}"), result)
             diagnostics.append((ref, result))
             analysis_statuses[task.id] = ("completed", None)
         insight_suite.record_analysis(analysis_statuses)

plugins/nemo-eval-author/tests/test_eval_author_agent.py

@@ -262,7 +262,6 @@ def validate_metric_contracts(
     monkeypatch.setattr(eval_author_module, "TraceAnalyzer", FakeTraceAnalyzer)
     monkeypatch.setattr(eval_author_module, "validate_metric_contracts", validate_metric_contracts)
 
-    monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None)
     monkeypatch.setattr(eval_author_module, "doc", lambda *_args, **_kwargs: "dataset docs")
     return calls
 

plugins/nemo-eval-author/tests/test_plugin_boundary.py

@@ -29,7 +29,6 @@
 # adding one needs a deliberate argument for why duplicating the helper is worse.
 #
 #   client                  -> make_client, the platform client factory
-#   ...components           -> the cache module, for run artifacts
 #   ...dataset_staging      -> stage_eval_author_inputs
 #   ...evaluator.base       -> EvaluatorType
 #   ...evaluator.factory    -> DatasetFactory
@@ -41,7 +40,6 @@
 #   ...reporting            -> RunReporter (ASE-749: reuse Experimentalist narrator; do not duplicate)
 _BORROWED_BEHAVIOUR = {
     "nemo_experimentalist_plugin.client",
-    "nemo_experimentalist_plugin.experimentalist.components",
     "nemo_experimentalist_plugin.experimentalist.components.dataset_staging",
     "nemo_experimentalist_plugin.experimentalist.components.evaluator.base",
     "nemo_experimentalist_plugin.experimentalist.components.evaluator.factory",

plugins/nemo-experimentalist/examples/smoke-agent/.gitignore

@@ -0,0 +1,7 @@
+# Generated by scripts/build_all_group.py, not authored.
+#
+# The loop takes a single --train-dataset path, so running several groups at once
+# needs one combined directory. Its contents are byte-identical copies of the
+# other groups, so committing it would double the dataset in the repo and put
+# every group change in two places. Build it before running the full scenario.
+dataset/groups/_all/

plugins/nemo-experimentalist/examples/smoke-agent/AGENT-SPEC.md

@@ -0,0 +1,75 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# smoke-agent
+
+## Prerequisites
+
+The task container ships Python, the standard library, and NOOA. It has no
+network access and no API key, so anything that needs either fails outright.
+
+## Job
+
+Answer one question about the records file at `/app/data/records.json` and write
+the single answer line to `/app/artifacts/output.txt`.
+
+## Interface
+
+- Invoked as `python main.py --prompt "<instruction text>"` with `/app` as the
+  working directory.
+- Writes exactly one line, plus a trailing newline, to
+  `/app/artifacts/output.txt`.
+- Writes an OTLP JSONL trace under `/app/traces/`.
+
+## Design
+
+`ReportAgent.solve` dispatches the instruction across an ordered list of
+handlers and returns the first non-`None` answer, falling back to a fixed
+string. Each handler matches the question with a regular expression, looks the
+answer up in the records, and formats one line as `<field>=<value>`.
+
+The records are a list of objects with `name`, `dept`, `role`, and `hours`.
+`FIELD_ALIASES` maps the word a question uses to the key the records store it
+under, so the answer line is always keyed by the canonical field name.
+
+## Missing and empty values
+
+A question may name a person the records do not contain, or ask for a field
+whose stored value is an empty string. Both are answered the same way: the value
+is the word `unknown`, so the line reads `dept=unknown`. This is part of the
+output contract and is compared byte-for-byte like any other answer — the
+sentinel is `unknown` exactly, not `n/a`, `none`, or the empty string.
+
+## Answer keys
+
+The key on the left of the `=` names what the answer *is*, not the field it came
+from. The vocabulary is fixed:
+
+- a value read from one record uses that field's own name — `dept=`, `role=`,
+  `hours=`
+- a sum over records is reported as **`total=`**, whatever field was summed and
+  however the records were selected
+- a number of records is reported as `count=`
+
+Keys are compared byte-for-byte like the rest of the line, so `hours=99` is wrong
+where `total=99` is expected, even though the number is right.
+
+## Constraints — these are hard requirements
+
+- **The agent is deterministic and offline.** The same instruction must always
+  produce the same answer. Reward differences between candidates must come from
+  code changes, never from sampling.
+- **No LLM.** Do not add a `@strategy` method, an LLM-backed handler, a subagent
+  with its own model, or a model swap. The task container has no network and no
+  API key, so such a change fails outright — but more importantly, being
+  reproducible is this agent's entire contract.
+- Standard library plus NOOA only. No new dependencies.
+- Do not edit `/app/data/records.json`. It is task-supplied input, not agent
+  code, and it is not part of this directory.
+- The output line is compared byte-for-byte against the task's expected value,
+  so trailing whitespace, extra lines, and changes to the `<field>=<value>` form
+  all count as wrong answers.
+
+## Next steps
+
+Any implementation change must still satisfy every constraint above.