From f0db9bef43661b383868b1699a6b26412a0bd461 Mon Sep 17 00:00:00 2001 From: Mayk Thewessen Date: Thu, 18 Jun 2026 17:50:55 +0200 Subject: [PATCH] MASTR: load any dated open-mastr dump (date-agnostic paths + glob fn) The loader hardcoded the 2025-02-09 dump's internal folder, so pointing the source at any newer dated dump raised KeyError on the storage-units read. Make the loader date-agnostic: - data.py: match bnetza_mastr_storage_units_raw.csv by suffix instead of the hardcoded bnetza_open_mastr_2025-02-09/ folder prefix. - data.py: fall back to KwkMastrNummer alone for CHP detection when ThermischeNutzleistung is absent (present in the Zenodo CSV dump, missing from the open-mastr bulk export). - utils.py get_raw_file: support a glob fn, selecting the newest matching local dump and falling back to the URL basename when none exists. - config.yaml: default MASTR.fn to the glob bnetza_open_mastr_*.zip, with the frozen Zenodo url kept as the download seed. Behaviour-preserving on the current Zenodo dump. Addresses #299. --- powerplantmatching/data.py | 18 +++++++++++++++--- powerplantmatching/package_data/config.yaml | 7 ++++++- powerplantmatching/utils.py | 21 ++++++++++++++++++++- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 524c15b..af59c31 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -2408,8 +2408,12 @@ def MASTR( cols = ["NutzbareSpeicherkapazitaet", "VerknuepfteEinheit"] with ZipFile(fn, "r") as file: - fn_storage_units = ( - "bnetza_open_mastr_2025-02-09/bnetza_mastr_storage_units_raw.csv" + # Match by suffix rather than a hardcoded dated folder, so newer + # open-mastr dumps (different bnetza_open_mastr_/ prefix) load. + fn_storage_units = next( + name + for name in file.namelist() + if name.endswith("bnetza_mastr_storage_units_raw.csv") ) storage_units = pd.read_csv(file.open(fn_storage_units), usecols=cols) @@ -2473,8 +2477,16 @@ def MASTR( parse_columns=PARSE_COLUMNS, ) .assign( + # ThermischeNutzleistung is present in the Zenodo CSV dump but absent + # from the open-mastr bulk export; fall back to KwkMastrNummer alone. Set=lambda df: df["Set"].where( - df["KwkMastrNummer"].isna() & df["ThermischeNutzleistung"].isna(), "CHP" + df["KwkMastrNummer"].isna() + & ( + df["ThermischeNutzleistung"].isna() + if "ThermischeNutzleistung" in df.columns + else True + ), + "CHP", ), ) ) diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index aced4f3..343d9ba 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -226,7 +226,12 @@ MASTR: reliability_score: 7 status: ["In Betrieb", "In Planung", "Endgültig stillgelegt", "Vorübergehend stillgelegt"] capacity_threshold: 0.1 # all values below will be filtered out, given in MW - fn: bnetza_open_mastr_2025-02-09.zip + # fn may be a glob: when it contains a wildcard the most recent matching + # local bnetza_open_mastr_.zip is auto-selected (ISO dates sort + # chronologically, so the newest wins). url is the frozen Zenodo seed, + # downloaded only when no local match exists, so it never clobbers a newer + # local build. Use a literal filename to pin one specific dump instead. + fn: bnetza_open_mastr_*.zip url: https://zenodo.org/records/14783581/files/bnetza_open_mastr_2025-02-09.zip EESI: net_capacity: true diff --git a/powerplantmatching/utils.py b/powerplantmatching/utils.py index 4b9396b..db2495d 100644 --- a/powerplantmatching/utils.py +++ b/powerplantmatching/utils.py @@ -10,7 +10,9 @@ import os import re from ast import literal_eval as liteval +from glob import glob from importlib.metadata import version +from urllib.parse import urlparse import country_converter as coco import numpy as np @@ -76,7 +78,24 @@ def get_raw_file(name, update=False, config=None, skip_retrieve=False): if config is None: config = get_config() df_config = config[name] - path = _data_in(df_config["fn"]) + fn = df_config["fn"] + + # A glob pattern in `fn` selects the most recent matching local file + # (e.g. a locally built, dated dump). ISO-dated filenames sort + # chronologically, so the last match is the newest. Falls back to + # downloading the URL's basename when nothing local matches. + if any(c in fn for c in "*?["): + matches = sorted(glob(_data_in(fn))) + if matches: + # The newest local dated build is authoritative. The URL is a + # frozen seed (e.g. the last Zenodo dump) that can never be newer + # than a local build, so honour the local match even when + # update=True; otherwise a forced refresh silently regresses to + # stale data. + return matches[-1] + path = _data_in(os.path.basename(urlparse(df_config["url"]).path)) + else: + path = _data_in(fn) if (not os.path.exists(path) or update) and not skip_retrieve: url = df_config["url"]