diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index fbe80c7..cc96df1 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,7 +524,9 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols].div(df["Capacity"], axis=0).where(lambda df: df != 0) + **df[weighted_cols] + .div(df["Capacity"].replace(0, pd.NA), axis=0) + .where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 08fd526..a7053e4 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -483,6 +483,90 @@ def set_large_spanish_stores_to_reservoirs(df): return df +def JRC_PPDB_OPEN(raw=False, update=False, config=None): + """ + Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN). + + Published by the European Commission's Joint Research Centre + (DOI: 10.5281/zenodo.3574566), this database was created + specifically to link ENTSO-E EIC codes with geographic + coordinates. It covers ~70% of large European power plants + and provides a deterministic bridge between EIC-based + operational data (ENTSO-E) and spatial data (OSM/GEM/GEO). + + Parameters + ---------- + raw : bool, default False + Whether to return the original dataset + update : bool, default False + Whether to update the data from the URL + config : dict, default None + Custom configuration + """ + config = get_config() if config is None else config + + fn = get_raw_file("JRC-PPDB-OPEN", update, config) + + from zipfile import ZipFile + + with ZipFile(fn, "r") as zf: + with zf.open("JRC_OPEN_UNITS.csv") as f: + jrc = pd.read_csv(f) + + if raw: + return jrc + + jrc = jrc[jrc["eic_p"].notna() & jrc["lat"].notna() & jrc["lon"].notna()] + + # Aggregate generation units to production units + jrc_map = ( + jrc.groupby("eic_p") + .agg( + { + "lat": "mean", + "lon": "mean", + "name_p": "first", + "capacity_p": "sum", + "type_g": "first", + "country": "first", + } + ) + .reset_index() + ) + + df = pd.DataFrame() + df["Name"] = jrc_map["name_p"] + df["Fueltype"] = jrc_map["type_g"] + df["Country"] = jrc_map["country"] + df["Capacity"] = jrc_map["capacity_p"] + df["lat"] = jrc_map["lat"] + df["lon"] = jrc_map["lon"] + df["EIC"] = jrc_map["eic_p"] + df["projectID"] = jrc_map["eic_p"] + + for col in [ + "Technology", + "Set", + "Efficiency", + "DateIn", + "DateRetrofit", + "DateOut", + "Duration", + "Volume_Mm3", + "DamHeight_m", + "StorageCapacity_MWh", + ]: + df[col] = None + + df = df[df["Capacity"].notna() & (df["Capacity"] > 0)] + + return ( + df.pipe(clean_name) + .pipe(set_column_name, "JRC-PPDB-OPEN") + .pipe(config_filter, config) + ) + + @deprecated( deprecated_in="0.5.0", details="Use the JRC data instead", diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py new file mode 100644 index 0000000..2edf538 --- /dev/null +++ b/powerplantmatching/eic_codes.py @@ -0,0 +1,47 @@ +"""EIC (Energy Identification Code) utilities. + +EIC codes are 16-character identifiers assigned by ENTSO-E. +The third character denotes the object type: 'W' = Resource Object +(power plants and generation units). +""" + +import re + +import pandas as pd + +# ENTSO-E EIC code pattern: 16 chars, 3rd char = 'W' (Resource Object) +EIC_PATTERN = re.compile(r"^..W.{13}$") + + +def is_valid_eic(code: str | None) -> bool: + """Return True if *code* is a valid EIC code.""" + if code is None: + return False + return bool(EIC_PATTERN.match(str(code))) + + +def extract_eics(series: pd.Series) -> dict[str, list]: + """Extract valid EIC codes from a pandas Series. + + Handles scalar strings and sets (as stored after aggregation). + + Returns + ------- + dict + ``{eic_code: [index, ...]}`` — each valid EIC mapped to the + list of row indices where it appears. + """ + result: dict[str, list] = {} + for idx, val in series.items(): + try: + if pd.isna(val): + continue + except (ValueError, TypeError): + pass + if isinstance(val, set): + for v in val: + if isinstance(v, str) and EIC_PATTERN.match(v): + result.setdefault(v, []).append(idx) + elif isinstance(val, str) and EIC_PATTERN.match(val): + result.setdefault(val, []).append(idx) + return result diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 0d95018..ca0ae0a 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -80,7 +80,7 @@ def compare_two_datasets(dfs, labels, country_wise=True, config=None, **dukeargs def country_link(dfs, country): # country_selector for both dataframes sel_country_b = [df["Country"] == country for df in dfs] - # only append if country appears in both dataframse + # only append if country appears in both dataframes if all(sel.any() for sel in sel_country_b): return duke( [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs @@ -287,7 +287,12 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): "DateRetrofit": "max", "DateOut": "max", "projectID": lambda x: dict(x.droplevel(0).dropna()), - "eic_code": set, + "EIC": lambda x: set( + v + for val in x.dropna() + for v in (val if isinstance(val, set) else [val]) + if isinstance(v, str) + ), } ) props_for_groups = pd.Series(props_for_groups)[cols].to_dict() diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index aced4f3..e8993c9 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -24,6 +24,7 @@ matching_sources: # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear' - JRC: Capacity >= 1 + - JRC-PPDB-OPEN: Capacity >= 1 # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - BEYONDCOAL @@ -48,6 +49,7 @@ fully_included_sources: - BEYONDCOAL # include this selection of countries as they have poorer coverage in all other datasets - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia'] + - JRC-PPDB-OPEN # these sources skip unit aggregation for fully_included_sources not covered in matching_sources aggregate_only_matching_sources: @@ -96,6 +98,10 @@ JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv +JRC-PPDB-OPEN: + reliability_score: 5 + fn: JRC-PPDB-OPEN.ver1.0.zip + url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip GEO: net_capacity: false reliability_score: 2