From ae1cb0f72a9cfe206795d4394f99e3378e59ee03 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Tue, 9 Jun 2026 15:38:33 +0200 Subject: [PATCH 1/4] extend verification results with truth label --- workflow/rules/common.smk | 10 ++++++++-- workflow/rules/verification.smk | 12 ++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index dc7fa0fc..009fc6a6 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -273,10 +273,16 @@ def resolve_baseline_id(label: str) -> str: def collect_experiment_participants(): participants = {} for base in BASELINE_CONFIGS.keys(): - participants[base] = OUT_ROOT / f"data/baselines/{base}/verif_aggregated.nc" + participants[base] = ( + OUT_ROOT + / f"data/baselines/{base}/verif_aggregated_{config['truth']['label']}.nc" + ) for exp in RUN_CONFIGS.keys(): if RUN_CONFIGS[exp].get("_is_candidate", False): - participants[exp] = OUT_ROOT / f"data/runs/{exp}/verif_aggregated.nc" + participants[exp] = ( + OUT_ROOT + / f"data/runs/{exp}/verif_aggregated_{config['truth']['label']}.nc" + ) return participants diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 394eaf87..58920063 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -16,9 +16,10 @@ rule verification_metrics_baseline: script="workflow/scripts/verification_metrics.py", forecast=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["root"], truth=config["truth"]["root"], + truth_label=config["truth"]["label"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc", + OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif_{truth_label}.nc", log: OUT_ROOT / "logs/verification_metrics_baseline/{baseline_id}-{init_time}.log", resources: @@ -67,7 +68,8 @@ rule verification_metrics: truth=config["truth"]["root"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc", + OUT_ROOT + / f"data/runs/{{run_id}}/{{init_time}}/verif_{config['truth']['label']}.nc", log: OUT_ROOT / "logs/verification_metrics/{run_id}-{init_time}.log", resources: @@ -120,7 +122,8 @@ rule verification_metrics_aggregation: allow_missing=True, ), output: - OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", + OUT_ROOT + / f"data/runs/{{run_id}}/verif_aggregated_{config['truth']['label']}.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation/{run_id}.log", resources: @@ -142,7 +145,8 @@ use rule verification_metrics_aggregation as verification_metrics_aggregation_ba allow_missing=True, ), output: - OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + OUT_ROOT + / f"data/baselines/{{baseline_id}}/verif_aggregated_{config['truth']['label']}.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation_baseline/{baseline_id}.log", From 6bb4bb7a839090e53447a3a500a394d8d2007a31 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Tue, 9 Jun 2026 15:39:06 +0200 Subject: [PATCH 2/4] add truth label and root to verification NetCDF metadata --- workflow/scripts/verification_metrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/scripts/verification_metrics.py b/workflow/scripts/verification_metrics.py index 0281d1c0..f549042f 100644 --- a/workflow/scripts/verification_metrics.py +++ b/workflow/scripts/verification_metrics.py @@ -85,6 +85,8 @@ def main(args: ScriptConfig): ) # save results to NetCDF + results.attrs["truth_label"] = args.truth_label + results.attrs["truth_root"] = str(args.truth) args.output.parent.mkdir(parents=True, exist_ok=True) results.earthkit.to_netcdf(args.output) LOG.info("Saved verification results to %s", args.output) From e195a32717ccb80069872605a28d75fdf0a4a04f Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Tue, 9 Jun 2026 22:07:27 +0200 Subject: [PATCH 3/4] Revert "extend verification results with truth label" This reverts commit ae1cb0f72a9cfe206795d4394f99e3378e59ee03. --- workflow/rules/common.smk | 10 ++-------- workflow/rules/verification.smk | 12 ++++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 009fc6a6..dc7fa0fc 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -273,16 +273,10 @@ def resolve_baseline_id(label: str) -> str: def collect_experiment_participants(): participants = {} for base in BASELINE_CONFIGS.keys(): - participants[base] = ( - OUT_ROOT - / f"data/baselines/{base}/verif_aggregated_{config['truth']['label']}.nc" - ) + participants[base] = OUT_ROOT / f"data/baselines/{base}/verif_aggregated.nc" for exp in RUN_CONFIGS.keys(): if RUN_CONFIGS[exp].get("_is_candidate", False): - participants[exp] = ( - OUT_ROOT - / f"data/runs/{exp}/verif_aggregated_{config['truth']['label']}.nc" - ) + participants[exp] = OUT_ROOT / f"data/runs/{exp}/verif_aggregated.nc" return participants diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 58920063..394eaf87 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -16,10 +16,9 @@ rule verification_metrics_baseline: script="workflow/scripts/verification_metrics.py", forecast=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["root"], truth=config["truth"]["root"], - truth_label=config["truth"]["label"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif_{truth_label}.nc", + OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc", log: OUT_ROOT / "logs/verification_metrics_baseline/{baseline_id}-{init_time}.log", resources: @@ -68,8 +67,7 @@ rule verification_metrics: truth=config["truth"]["root"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT - / f"data/runs/{{run_id}}/{{init_time}}/verif_{config['truth']['label']}.nc", + OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc", log: OUT_ROOT / "logs/verification_metrics/{run_id}-{init_time}.log", resources: @@ -122,8 +120,7 @@ rule verification_metrics_aggregation: allow_missing=True, ), output: - OUT_ROOT - / f"data/runs/{{run_id}}/verif_aggregated_{config['truth']['label']}.nc", + OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation/{run_id}.log", resources: @@ -145,8 +142,7 @@ use rule verification_metrics_aggregation as verification_metrics_aggregation_ba allow_missing=True, ), output: - OUT_ROOT - / f"data/baselines/{{baseline_id}}/verif_aggregated_{config['truth']['label']}.nc", + OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation_baseline/{baseline_id}.log", From a667b01afaeedea0903f5a970ecb6296f154dcb0 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Tue, 9 Jun 2026 23:21:39 +0200 Subject: [PATCH 4/4] add truth label to verification data --- workflow/rules/common.smk | 9 +++++++-- workflow/rules/verification.smk | 9 +++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index dc7fa0fc..7e154e5e 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -8,6 +8,7 @@ from urllib.parse import urlparse CONFIG_ROOT = Path("config").resolve() OUT_ROOT = Path(config["locations"]["output_root"]) +TRUTH_LABEL = config["truth"]["label"] DATETIME_FORMAT = "%Y-%m-%dT%H:%M" HASH_LENGTH = 4 @@ -273,10 +274,14 @@ def resolve_baseline_id(label: str) -> str: def collect_experiment_participants(): participants = {} for base in BASELINE_CONFIGS.keys(): - participants[base] = OUT_ROOT / f"data/baselines/{base}/verif_aggregated.nc" + participants[base] = ( + OUT_ROOT / f"data/baselines/{base}/verif_aggregated_{TRUTH_LABEL}.nc" + ) for exp in RUN_CONFIGS.keys(): if RUN_CONFIGS[exp].get("_is_candidate", False): - participants[exp] = OUT_ROOT / f"data/runs/{exp}/verif_aggregated.nc" + participants[exp] = ( + OUT_ROOT / f"data/runs/{exp}/verif_aggregated_{TRUTH_LABEL}.nc" + ) return participants diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 394eaf87..530645ff 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -18,7 +18,8 @@ rule verification_metrics_baseline: truth=config["truth"]["root"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc", + OUT_ROOT + / f"data/baselines/{{baseline_id}}/{{init_time}}/verif_{TRUTH_LABEL}.nc", log: OUT_ROOT / "logs/verification_metrics_baseline/{baseline_id}-{init_time}.log", resources: @@ -67,7 +68,7 @@ rule verification_metrics: truth=config["truth"]["root"], eckit_grids=rules.data_download_eckit_geo_grids.output, output: - OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc", + OUT_ROOT / f"data/runs/{{run_id}}/{{init_time}}/verif_{TRUTH_LABEL}.nc", log: OUT_ROOT / "logs/verification_metrics/{run_id}-{init_time}.log", resources: @@ -120,7 +121,7 @@ rule verification_metrics_aggregation: allow_missing=True, ), output: - OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", + OUT_ROOT / f"data/runs/{{run_id}}/verif_aggregated_{TRUTH_LABEL}.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation/{run_id}.log", resources: @@ -142,7 +143,7 @@ use rule verification_metrics_aggregation as verification_metrics_aggregation_ba allow_missing=True, ), output: - OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + OUT_ROOT / f"data/baselines/{{baseline_id}}/verif_aggregated_{TRUTH_LABEL}.nc", log: OUT_ROOT / "logs/verification_metrics_aggregation_baseline/{baseline_id}.log",