From 0416bd0bc9c381cb00473ab4ea06dff841aba9a4 Mon Sep 17 00:00:00 2001 From: rioloc Date: Tue, 25 Aug 2026 10:56:16 +0200 Subject: [PATCH] feat: remove Retrying params from Openshift Agentic evaluation --- docs/agentic_lightspeed_evaluation.md | 5 +- .../custom/openshift_agentic_run_eval.py | 27 ------ .../core/openshift_agentic_run/phase.py | 7 +- .../pipeline/evaluation/driver.py | 2 +- ...evaluation_data_openshift_agentic_run.yaml | 1 - .../custom/test_openshift_agentic_run_eval.py | 17 +++- ...t_openshift_agentic_run_eval_assertions.py | 91 +------------------ .../test_openshift_agentic_run_driver.py | 9 +- 8 files changed, 26 insertions(+), 133 deletions(-) diff --git a/docs/agentic_lightspeed_evaluation.md b/docs/agentic_lightspeed_evaluation.md index 53934e85..edf53f36 100644 --- a/docs/agentic_lightspeed_evaluation.md +++ b/docs/agentic_lightspeed_evaluation.md @@ -121,6 +121,7 @@ The simplest agentic evaluation — analysis phase only, no execution or verific analysis: agent: eval-default expected_openshift_agentic_run_status: + max_duration: "15m" phase: Completed turn_metrics: - custom:openshift_agentic_run_status @@ -157,8 +158,6 @@ Complete remediation workflow with deterministic assertions and LLM-as-judge: agent: eval-default expected_openshift_agentic_run_status: phase: Completed - max_duration: "15m" - max_attempts: 5 analysis: min_options: 1 options: @@ -217,7 +216,6 @@ Checks run in order: **phase → timing → analysis → execution → verificat | Field | Type | Description | |-------|------|-------------| | `max_duration` | string | Max elapsed time across conditions. Go-style duration: `"5m"`, `"2m30s"`, `"1h"` | -| `max_attempts` | int | Max number of execution attempts. Read from `status.attempts` or inferred from `RetryingExecution` conditions | **Analysis checks:** @@ -254,7 +252,6 @@ Checks run in order: **phase → timing → analysis → execution → verificat | `conditions[].status` | string | Expected condition status (e.g., `"True"`, `"False"`) | | `conditions[].reason` | string | Expected condition reason (e.g., `Skipped`, `Succeeded`) | -> On retried AgenticRuns, analysis and execution checks use the **latest** (most recent) Result CR, so assertions reflect the final execution state. ### `custom:openshift_agentic_run_evaluation_correctness` — LLM-as-Judge diff --git a/src/lightspeed_evaluation/core/metrics/custom/openshift_agentic_run_eval.py b/src/lightspeed_evaluation/core/metrics/custom/openshift_agentic_run_eval.py index ba264d8b..77e54085 100644 --- a/src/lightspeed_evaluation/core/metrics/custom/openshift_agentic_run_eval.py +++ b/src/lightspeed_evaluation/core/metrics/custom/openshift_agentic_run_eval.py @@ -143,32 +143,6 @@ def _check_max_duration( return False, f"Duration {elapsed:.0f}s exceeds limit {max_duration} ({limit:.0f}s)" -def _check_max_attempts( - expected: dict[str, Any], - conditions: list[dict[str, Any]], - openshift_agentic_run_status: dict[str, Any], -) -> Optional[tuple[bool, str]]: - """Check that the number of execution attempts is within limit.""" - max_attempts = expected.get("max_attempts") - if max_attempts is None: - return None - - actual = openshift_agentic_run_status.get("attempts") - if actual is None: - actual = ( - sum( - 1 - for c in conditions - if isinstance(c, dict) and c.get("reason") == "RetryingExecution" - ) - + 1 - ) - - if actual <= max_attempts: - return True, f"Attempts {actual} within limit {max_attempts}" - return False, f"Attempts {actual} exceeds limit {max_attempts}" - - def _check_analysis_component( comp_type: str, expected_comp: dict[str, Any], @@ -465,7 +439,6 @@ def evaluate_openshift_agentic_run_status( _check_phase(expected, conditions, openshift_agentic_run_spec), _check_phase_in(expected, conditions, openshift_agentic_run_spec), _check_max_duration(expected, conditions), - _check_max_attempts(expected, conditions, openshift_agentic_run_status), _check_analysis(expected, openshift_agentic_run_results), _check_execution(expected, openshift_agentic_run_results), _check_conditions(expected, conditions), diff --git a/src/lightspeed_evaluation/core/openshift_agentic_run/phase.py b/src/lightspeed_evaluation/core/openshift_agentic_run/phase.py index e519e486..3b5637d4 100644 --- a/src/lightspeed_evaluation/core/openshift_agentic_run/phase.py +++ b/src/lightspeed_evaluation/core/openshift_agentic_run/phase.py @@ -14,7 +14,7 @@ def derive_phase( openshift_agentic_run_spec: AgenticRun spec to determine the last expected step. Returns: - Phase string: Completed, Failed, Denied, Escalated, or InProgress. + Phase string: Completed, Failed, Denied, Escalated, Escalating, or InProgress. """ by_type = {c["type"]: c for c in conditions if isinstance(c, dict) and "type" in c} @@ -23,11 +23,14 @@ def derive_phase( if by_type.get("Escalated", {}).get("status") == "True": return "Escalated" + escalated = by_type.get("Escalated", {}) + if escalated.get("status") == "Unknown": + return "Escalating" + for c in conditions: if isinstance(c, dict) and ( c.get("type") in {"Analyzed", "Executed", "Verified"} and c.get("status") == "False" - and c.get("reason") != "RetryingExecution" ): return "Failed" diff --git a/src/lightspeed_evaluation/pipeline/evaluation/driver.py b/src/lightspeed_evaluation/pipeline/evaluation/driver.py index e1eb7212..2a4c54b6 100644 --- a/src/lightspeed_evaluation/pipeline/evaluation/driver.py +++ b/src/lightspeed_evaluation/pipeline/evaluation/driver.py @@ -117,7 +117,7 @@ class TerminalOutcome(StrEnum): - Denied: True = user denied a step (terminal) - Escalated: True = escalation complete (terminal), False = failed, Unknown = in progress - Special reason: RetryingExecution (Verified=False triggers retry, not failure). + Verification failure now escalates directly (no retry mechanism). """ COMPLETED = "Completed" diff --git a/tests/integration/test_evaluation_data_openshift_agentic_run.yaml b/tests/integration/test_evaluation_data_openshift_agentic_run.yaml index d7c07696..e1a4df78 100644 --- a/tests/integration/test_evaluation_data_openshift_agentic_run.yaml +++ b/tests/integration/test_evaluation_data_openshift_agentic_run.yaml @@ -110,7 +110,6 @@ expected_openshift_agentic_run_status: phase: Completed max_duration: "15m" - max_attempts: 5 analysis: min_options: 1 execution: diff --git a/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval.py b/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval.py index 0bd7d5af..84eb82f1 100644 --- a/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval.py +++ b/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval.py @@ -90,14 +90,23 @@ def test_failed_condition(self) -> None: ] assert derive_phase(conditions) == "Failed" - def test_retrying_execution_not_failed(self) -> None: - """RetryingExecution reason does not count as failure.""" + def test_verification_failure_is_failed(self) -> None: + """Verified=False now maps to Failed (no retry mechanism).""" conditions = [ {"type": "Analyzed", "status": "True"}, - {"type": "Verified", "status": "False", "reason": "RetryingExecution"}, + {"type": "Verified", "status": "False", "reason": "VerificationFailed"}, ] spec: dict[str, Any] = {"analysis": {}, "execution": {}, "verification": {}} - assert derive_phase(conditions, spec) == "InProgress" + assert derive_phase(conditions, spec) == "Failed" + + def test_escalating_phase(self) -> None: + """Escalated=Unknown derives Escalating (verification failure escalation).""" + conditions = [ + {"type": "Analyzed", "status": "True"}, + {"type": "Verified", "status": "False", "reason": "VerificationFailed"}, + {"type": "Escalated", "status": "Unknown", "reason": "VerificationFailed"}, + ] + assert derive_phase(conditions) == "Escalating" def test_denied(self) -> None: """Denied=True derives Denied.""" diff --git a/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval_assertions.py b/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval_assertions.py index 236c6752..b0ad829a 100644 --- a/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval_assertions.py +++ b/tests/unit/core/metrics/custom/test_openshift_agentic_run_eval_assertions.py @@ -1,6 +1,6 @@ """Unit tests for agentic run status assertion checks. -Covers: _parse_duration, max_duration, max_attempts, analysis +Covers: _parse_duration, max_duration, analysis (with component/option helpers), execution, and check ordering. """ @@ -169,92 +169,6 @@ def test_boundary_equal(self) -> None: assert "within limit" in reason -class TestMaxAttemptsCheck: - """Max attempts assertion.""" - - def test_within_limit_pass(self) -> None: - """Attempts within limit returns 1.0.""" - turn = _make_turn( - expected_openshift_agentic_run_status={"max_attempts": 3}, - openshift_agentic_run_status={ - "attempts": 2, - "conditions": [{"type": "Analyzed", "status": "True"}], - }, - ) - score, reason = evaluate_openshift_agentic_run_status(None, 0, turn, False) - assert score == 1.0 - assert "Attempts 2 within limit 3" in reason - - def test_exceeded_fail(self) -> None: - """Attempts exceeding limit returns 0.0.""" - turn = _make_turn( - expected_openshift_agentic_run_status={"max_attempts": 3}, - openshift_agentic_run_status={ - "attempts": 4, - "conditions": [{"type": "Analyzed", "status": "True"}], - }, - ) - score, reason = evaluate_openshift_agentic_run_status(None, 0, turn, False) - assert score == 0.0 - assert "exceeds limit" in reason - - def test_from_status_field(self) -> None: - """Reads attempts from openshift_agentic_run_status.attempts when available.""" - turn = _make_turn( - expected_openshift_agentic_run_status={"max_attempts": 5}, - openshift_agentic_run_status={ - "attempts": 1, - "conditions": [ - { - "type": "Executed", - "status": "False", - "reason": "RetryingExecution", - }, - {"type": "Analyzed", "status": "True"}, - ], - }, - ) - score, reason = evaluate_openshift_agentic_run_status(None, 0, turn, False) - assert score == 1.0 - assert "Attempts 1" in reason - - def test_inferred_from_conditions(self) -> None: - """Infers attempts from RetryingExecution conditions + 1.""" - turn = _make_turn( - expected_openshift_agentic_run_status={"max_attempts": 3}, - openshift_agentic_run_status={ - "conditions": [ - { - "type": "Executed", - "status": "False", - "reason": "RetryingExecution", - }, - { - "type": "Verified", - "status": "False", - "reason": "RetryingExecution", - }, - {"type": "Analyzed", "status": "True"}, - ], - }, - ) - score, reason = evaluate_openshift_agentic_run_status(None, 0, turn, False) - assert score == 1.0 - assert "Attempts 3" in reason - - def test_skip_when_not_specified(self) -> None: - """No max_attempts in expected skips the check.""" - turn = _make_turn( - expected_openshift_agentic_run_status={"phase": "Completed"}, - openshift_agentic_run_status={ - "conditions": [{"type": "Analyzed", "status": "True"}], - }, - openshift_agentic_run_spec={"analysis": {}}, - ) - score, _ = evaluate_openshift_agentic_run_status(None, 0, turn, False) - assert score == 1.0 - - class TestAnalysisCheck: """Analysis assertion checks (options, risk, confidence, components).""" @@ -913,12 +827,10 @@ def test_all_new_checks_pass(self) -> None: turn = _make_turn( expected_openshift_agentic_run_status={ "max_duration": "10m", - "max_attempts": 3, "analysis": {"min_options": 1}, "execution": {"phase": "Succeeded"}, }, openshift_agentic_run_status={ - "attempts": 1, "conditions": [ { "type": "Analyzed", @@ -949,6 +861,5 @@ def test_all_new_checks_pass(self) -> None: score, reason = evaluate_openshift_agentic_run_status(None, 0, turn, False) assert score == 1.0 assert "Duration" in reason - assert "Attempts" in reason assert "Analysis assertions passed" in reason assert "Execution assertions passed" in reason diff --git a/tests/unit/pipeline/evaluation/test_openshift_agentic_run_driver.py b/tests/unit/pipeline/evaluation/test_openshift_agentic_run_driver.py index 3bd730c4..1d8610f8 100644 --- a/tests/unit/pipeline/evaluation/test_openshift_agentic_run_driver.py +++ b/tests/unit/pipeline/evaluation/test_openshift_agentic_run_driver.py @@ -172,12 +172,13 @@ class TestIsTerminal: # pylint: disable=too-few-public-methods SPEC_FULL, None, ), - # RetryingExecution — not a failure + # Escalating — verification failure triggers escalation ( [ _cond("Analyzed", "True"), _cond("Executed", "True"), - _cond("Verified", "False", "RetryingExecution"), + _cond("Verified", "False", "VerificationFailed"), + _cond("Escalated", "Unknown", "VerificationFailed"), ], SPEC_FULL, None, @@ -202,7 +203,7 @@ class TestIsTerminal: # pylint: disable=too-few-public-methods SPEC_FULL, TerminalOutcome.COMPLETED, ), - # Failed — any condition False (no RetryingExecution) + # Failed — any condition False ( [_cond("Analyzed", "False")], SPEC_FULL, @@ -248,7 +249,7 @@ class TestIsTerminal: # pylint: disable=too-few-public-methods "executing", "executed-not-terminal-full", "verifying", - "retrying-execution", + "escalating", "completed-analysis-only", "completed-with-exec", "completed-full",