Fix EIC code aggregation and integrate JRC-PPDB-OPEN data source

powerplantmatching/cleaning.py

@@ -524,7 +524,9 @@ def aggregate_units(
 
     df = (
         df.assign(
-            **df[weighted_cols].div(df["Capacity"], axis=0).where(lambda df: df != 0)
+            **df[weighted_cols]
+            .div(df["Capacity"].replace(0, pd.NA), axis=0)
+            .where(lambda df: df != 0)
         )
         .reset_index(drop=True)
         .pipe(clean_name)

powerplantmatching/data.py

@@ -483,6 +483,90 @@ def set_large_spanish_stores_to_reservoirs(df):
     return df
 
 
+def JRC_PPDB_OPEN(raw=False, update=False, config=None):
+    """
+    Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN).
+
+    Published by the European Commission's Joint Research Centre
+    (DOI: 10.5281/zenodo.3574566), this database was created
+    specifically to link ENTSO-E EIC codes with geographic
+    coordinates.  It covers ~70% of large European power plants
+    and provides a deterministic bridge between EIC-based
+    operational data (ENTSO-E) and spatial data (OSM/GEM/GEO).
+
+    Parameters
+    ----------
+    raw : bool, default False
+        Whether to return the original dataset
+    update : bool, default False
+        Whether to update the data from the URL
+    config : dict, default None
+        Custom configuration
+    """
+    config = get_config() if config is None else config
+
+    fn = get_raw_file("JRC-PPDB-OPEN", update, config)
+
+    from zipfile import ZipFile
+
+    with ZipFile(fn, "r") as zf:
+        with zf.open("JRC_OPEN_UNITS.csv") as f:
+            jrc = pd.read_csv(f)
+
+    if raw:
+        return jrc
+
+    jrc = jrc[jrc["eic_p"].notna() & jrc["lat"].notna() & jrc["lon"].notna()]
+
+    # Aggregate generation units to production units
+    jrc_map = (
+        jrc.groupby("eic_p")
+        .agg(
+            {
+                "lat": "mean",
+                "lon": "mean",
+                "name_p": "first",
+                "capacity_p": "sum",
+                "type_g": "first",
+                "country": "first",
+            }
+        )
+        .reset_index()
+    )
+
+    df = pd.DataFrame()
+    df["Name"] = jrc_map["name_p"]
+    df["Fueltype"] = jrc_map["type_g"]
+    df["Country"] = jrc_map["country"]
+    df["Capacity"] = jrc_map["capacity_p"]
+    df["lat"] = jrc_map["lat"]
+    df["lon"] = jrc_map["lon"]
+    df["EIC"] = jrc_map["eic_p"]
+    df["projectID"] = jrc_map["eic_p"]
+
+    for col in [
+        "Technology",
+        "Set",
+        "Efficiency",
+        "DateIn",
+        "DateRetrofit",
+        "DateOut",
+        "Duration",
+        "Volume_Mm3",
+        "DamHeight_m",
+        "StorageCapacity_MWh",
+    ]:
+        df[col] = None
+
+    df = df[df["Capacity"].notna() & (df["Capacity"] > 0)]
+
+    return (
+        df.pipe(clean_name)
+        .pipe(set_column_name, "JRC-PPDB-OPEN")
+        .pipe(config_filter, config)
+    )
+
+
 @deprecated(
     deprecated_in="0.5.0",
     details="Use the JRC data instead",

powerplantmatching/eic_codes.py

@@ -0,0 +1,47 @@
+"""EIC (Energy Identification Code) utilities.
+
+EIC codes are 16-character identifiers assigned by ENTSO-E.
+The third character denotes the object type: 'W' = Resource Object
+(power plants and generation units).
+"""
+
+import re
+
+import pandas as pd
+
+# ENTSO-E EIC code pattern: 16 chars, 3rd char = 'W' (Resource Object)
+EIC_PATTERN = re.compile(r"^..W.{13}$")
+
+
+def is_valid_eic(code: str | None) -> bool:
+    """Return True if *code* is a valid EIC code."""
+    if code is None:
+        return False
+    return bool(EIC_PATTERN.match(str(code)))
+
+
+def extract_eics(series: pd.Series) -> dict[str, list]:
+    """Extract valid EIC codes from a pandas Series.
+
+    Handles scalar strings and sets (as stored after aggregation).
+
+    Returns
+    -------
+    dict
+        ``{eic_code: [index, ...]}`` — each valid EIC mapped to the
+        list of row indices where it appears.
+    """
+    result: dict[str, list] = {}
+    for idx, val in series.items():
+        try:
+            if pd.isna(val):
+                continue
+        except (ValueError, TypeError):
+            pass
+        if isinstance(val, set):
+            for v in val:
+                if isinstance(v, str) and EIC_PATTERN.match(v):
+                    result.setdefault(v, []).append(idx)
+        elif isinstance(val, str) and EIC_PATTERN.match(val):
+            result.setdefault(val, []).append(idx)
+    return result

powerplantmatching/matching.py

@@ -80,7 +80,7 @@ def compare_two_datasets(dfs, labels, country_wise=True, config=None, **dukeargs
     def country_link(dfs, country):
         # country_selector for both dataframes
         sel_country_b = [df["Country"] == country for df in dfs]
-        # only append if country appears in both dataframse
+        # only append if country appears in both dataframes
         if all(sel.any() for sel in sel_country_b):
             return duke(
                 [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs
@@ -287,7 +287,12 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None):
             "DateRetrofit": "max",
             "DateOut": "max",
             "projectID": lambda x: dict(x.droplevel(0).dropna()),
-            "eic_code": set,
+            "EIC": lambda x: set(
+                v
+                for val in x.dropna()
+                for v in (val if isinstance(val, set) else [val])
+                if isinstance(v, str)
+            ),
         }
     )
     props_for_groups = pd.Series(props_for_groups)[cols].to_dict()

powerplantmatching/package_data/config.yaml

@@ -24,6 +24,7 @@ matching_sources:
   # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets
   - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype  == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear'
   - JRC: Capacity >= 1
+  - JRC-PPDB-OPEN: Capacity >= 1
   # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets
   - OPSD: not (Country == 'Germany' and Fueltype  == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas')
   - BEYONDCOAL
@@ -48,6 +49,7 @@ fully_included_sources:
   - BEYONDCOAL
   # include this selection of countries as they have poorer coverage in all other datasets
   - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia']
+  - JRC-PPDB-OPEN
 
 # these sources skip unit aggregation for fully_included_sources not covered in matching_sources
 aggregate_only_matching_sources:
@@ -96,6 +98,10 @@ JRC:
   reliability_score: 5
   fn: jrc-hydro-power-plant-database.csv
   url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv
+JRC-PPDB-OPEN:
+  reliability_score: 5
+  fn: JRC-PPDB-OPEN.ver1.0.zip
+  url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip
 GEO:
   net_capacity: false
   reliability_score: 2