Faster reading of outputs

energytool/base/idfobject_utils.py

@@ -246,7 +246,7 @@ def add_output_variable(
     idf: IDF,
     key_values: str | list,
     variables,
-    reporting_frequency: str = "Hourly",
+    reporting_frequency: str = "Timestep",
 ):
     """
     This function allows you to add output:variable object to an EnergyPlus IDF file.
@@ -260,7 +260,8 @@ def add_output_variable(
     :param variables: The names of the variables to output. This can be a single
         variable name (string) or a list of variable names (list of strings).
     :param reporting_frequency: The reporting frequency for the output
-        variables (e.g., "Hourly", "Daily", etc.). Default is "Hourly."
+        variables (e.g., "Hourly", "Daily", etc.). Default is "Timestep" of the
+        simulation.
     :return: None
 
     The function iterates through the specified key values and variables, checking if
@@ -288,11 +289,13 @@ def add_output_variable(
                 if key == "*":
                     del_output_variable(idf, var)
 
+                freq = getattr(idf, "output_frequency", reporting_frequency)
+
                 idf.newidfobject(
                     "OUTPUT:VARIABLE",
                     Key_Value=key,
                     Variable_Name=var,
-                    Reporting_Frequency=reporting_frequency,
+                    Reporting_Frequency=freq,
                 )
 
 

energytool/building.py

@@ -4,8 +4,9 @@
 import os
 import tempfile
 import shutil
+import time
 
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from copy import deepcopy
 from pathlib import Path
 
@@ -15,6 +16,9 @@
 from eppy.runner.run_functions import run
 import eppy.json_functions as json_functions
 
+import sqlite3
+import pandas as pd
+
 import energytool.base.idf_utils
 from energytool.base.parse_results import read_eplus_res
 from energytool.outputs import get_results
@@ -39,6 +43,7 @@ class SimuOpt(enum.Enum):
     OUTPUTS = "outputs"
     EPW_FILE = "epw_file"
     VERBOSE = "verbose"
+    OUTPUT_FREQUENCY = "OUTPUT_FREQUENCY"
 
 
 @contextmanager
@@ -53,7 +58,65 @@ def temporary_directory():
         yield temp_dir
 
     finally:
-        shutil.rmtree(temp_dir)
+        for _ in range(20):
+            try:
+                shutil.rmtree(temp_dir)
+                break
+            except PermissionError:
+                time.sleep(0.2)
+
+
+def ensure_sql_output(idf):
+    objs = idf.idfobjects["OUTPUT:SQLITE"]
+    if not objs:
+        idf.newidfobject("OUTPUT:SQLITE", Option_Type="SimpleAndTabular")
+
+
+def read_sql_timeseries(sql_path, ref_year=None, unify_frequency=True):
+
+    query = """
+    SELECT
+        t.Month,
+        t.Day,
+        t.Hour,
+        t.Minute,
+        rdd.KeyValue || ':' || rdd.Name || ' [' || rdd.Units || '](' || rdd.ReportingFrequency || ')' AS variable,
+        rd.Value
+    FROM ReportData rd
+    JOIN ReportDataDictionary rdd
+        ON rd.ReportDataDictionaryIndex = rdd.ReportDataDictionaryIndex
+    JOIN Time t
+        ON rd.TimeIndex = t.TimeIndex
+    """
+
+    with sqlite3.connect(sql_path) as conn:
+        df = pd.read_sql_query(query, conn)
+
+    if ref_year is None:
+        ref_year = 2000
+
+    dt = pd.to_datetime(
+        dict(
+            year=ref_year,
+            month=df.Month,
+            day=df.Day,
+            hour=df.Hour,
+            minute=df.Minute,
+        )
+    )
+
+    df["datetime"] = dt
+
+    df = df.pivot(index="datetime", columns="variable", values="Value")
+    df = df.sort_index()
+
+    if unify_frequency:
+        step = df.index.to_series().diff().dropna().mode()[0]
+        full_index = pd.date_range(df.index.min(), df.index.max(), freq=step)
+        df = df.reindex(full_index)
+        df = df.ffill()
+
+    return df
 
 
 class Building(Model):
@@ -200,6 +263,7 @@ def simulate(
         self,
         property_dict=None,
         simulation_options=None,
+        working_directory=None,
         idf_save_path=None,
         **simulation_kwargs,
     ) -> pd.DataFrame:
@@ -333,6 +397,12 @@ def simulate(
                 ),
             )
 
+        output_frequency = simulation_options.get(
+            SimuOpt.OUTPUT_FREQUENCY.value,
+            "Timestep",
+        )
+        working_idf.output_frequency = output_frequency
+
         # PRE-PROCESS
         system_list = [sys for sublist in working_syst.values() for sys in sublist]
         for system in system_list:
@@ -342,29 +412,37 @@ def simulate(
         if SimuOpt.VERBOSE.value not in simulation_options.keys():
             simulation_options[SimuOpt.VERBOSE.value] = "v"
 
+        ensure_sql_output(working_idf)
+
+        import gc
+
+        gc.collect()
+
         # SIMULATE
-        with temporary_directory() as temp_dir:
+        if working_directory is None:
+            context = temporary_directory()
+        else:
+            working_directory = Path(working_directory)
+            working_directory.mkdir(parents=True, exist_ok=True)
+            context = nullcontext(working_directory)
+
+        with context as temp_dir:
+
             working_idf.saveas((Path(temp_dir) / "in.idf").as_posix(), encoding="utf-8")
             idd_ref = working_idf.idd_version
             run(
                 idf=working_idf,
                 weather=epw_path,
-                output_directory=temp_dir.replace("\\", "/"),
+                output_directory=Path(temp_dir).as_posix(),
                 annual=False,
                 design_day=False,
-                idd=None,
-                epmacro=False,
-                expandobjects=False,
-                readvars=True,
-                output_prefix=None,
-                output_suffix=None,
-                version=False,
+                readvars=False,
                 verbose=simulation_options[SimuOpt.VERBOSE.value],
                 ep_version=f"{idd_ref[0]}-{idd_ref[1]}-{idd_ref[2]}",
             )
 
-            eplus_res = read_eplus_res(
-                Path(temp_dir) / "eplusout.csv", ref_year=ref_year
+            eplus_res = read_sql_timeseries(
+                Path(temp_dir) / "eplusout.sql", ref_year=ref_year
             )
 
             # Save IDF file after pre-process

tests/base/test_idfobject_utils.py

@@ -31,35 +31,35 @@ def test_add_output_zone_variable(self, toy_idf):
         add_output_variable(toy_idf, key_values="Zone_1", variables="Conso")
 
         to_test = [elmt["obj"] for elmt in toy_idf.idfobjects["Output:Variable"]]
-        ref = [["OUTPUT:VARIABLE", "Zone_1", "Conso", "Hourly"]]
+        ref = [["OUTPUT:VARIABLE", "Zone_1", "Conso", "Timestep"]]
         assert to_test == ref
 
         add_output_variable(toy_idf, key_values=["Zone_1", "Zone_2"], variables="Conso")
 
         to_test = [elmt["obj"] for elmt in toy_idf.idfobjects["Output:Variable"]]
         ref = [
-            ["OUTPUT:VARIABLE", "Zone_1", "Conso", "Hourly"],
-            ["OUTPUT:VARIABLE", "Zone_2", "Conso", "Hourly"],
+            ["OUTPUT:VARIABLE", "Zone_1", "Conso", "Timestep"],
+            ["OUTPUT:VARIABLE", "Zone_2", "Conso", "Timestep"],
         ]
         assert to_test == ref
 
         add_output_variable(toy_idf, key_values="Zone_3", variables=["Conso", "Elec"])
 
         to_test = [elmt["obj"] for elmt in toy_idf.idfobjects["Output:Variable"]]
         ref = [
-            ["OUTPUT:VARIABLE", "Zone_1", "Conso", "Hourly"],
-            ["OUTPUT:VARIABLE", "Zone_2", "Conso", "Hourly"],
-            ["OUTPUT:VARIABLE", "Zone_3", "Conso", "Hourly"],
-            ["OUTPUT:VARIABLE", "Zone_3", "Elec", "Hourly"],
+            ["OUTPUT:VARIABLE", "Zone_1", "Conso", "Timestep"],
+            ["OUTPUT:VARIABLE", "Zone_2", "Conso", "Timestep"],
+            ["OUTPUT:VARIABLE", "Zone_3", "Conso", "Timestep"],
+            ["OUTPUT:VARIABLE", "Zone_3", "Elec", "Timestep"],
         ]
         assert to_test == ref
 
         add_output_variable(toy_idf, key_values="*", variables="Conso")
 
         to_test = [elmt["obj"] for elmt in toy_idf.idfobjects["Output:Variable"]]
         ref = [
-            ["OUTPUT:VARIABLE", "Zone_3", "Elec", "Hourly"],
-            ["OUTPUT:VARIABLE", "*", "Conso", "Hourly"],
+            ["OUTPUT:VARIABLE", "Zone_3", "Elec", "Timestep"],
+            ["OUTPUT:VARIABLE", "*", "Conso", "Timestep"],
         ]
         assert to_test == ref
 

tests/test_building.py

@@ -21,7 +21,8 @@ def test_building(self):
             "epw_file": (RESOURCES_PATH / "B4R_weather_Paris_2020.epw").as_posix(),
         }
         simulation_options = {
-            SimuOpt.OUTPUTS.value: f"{OutputCategories.SYSTEM.value}|{OutputCategories.RAW.value}"
+            SimuOpt.OUTPUTS.value: f"{OutputCategories.SYSTEM.value}|{OutputCategories.RAW.value}",
+            SimuOpt.OUTPUT_FREQUENCY.value: "Hourly",
         }
 
         res = test_build.simulate(
@@ -62,7 +63,7 @@ def test_building(self):
                 "[J](Hourly)": 15276675722.295742,
                 "BLOCK2:APPTX2E IDEAL LOADS AIR:"
                 "Zone Ideal Loads Supply Air Total Heating Energy "
-                "[J](Hourly) ": 15677421945.907581,
+                "[J](Hourly)": 15677421945.907581,
             },
             rel=0.05,
         )
@@ -84,6 +85,7 @@ def test_boundaries(self):
             SimuOpt.OUTPUTS.value: f""
             f"{OutputCategories.SYSTEM.value}|"
             f"{OutputCategories.RAW.value}",
+            SimuOpt.OUTPUT_FREQUENCY.value: "Hourly",
         }
 
         res = test_build.simulate(
@@ -117,7 +119,7 @@ def test_boundaries(self):
                 "[J](Hourly)": 3571575271.9865627,
                 "BLOCK2:APPTX2E IDEAL LOADS AIR:"
                 "Zone Ideal Loads Supply Air Total Heating Energy "
-                "[J](Hourly) ": 3636379021.517704,
+                "[J](Hourly)": 3636379021.517704,
             },
             rel=0.05,
         )

tests/test_system.py

@@ -146,19 +146,19 @@ def test_heater_simple(self, idf):
                 "OUTPUT:VARIABLE",
                 "*",
                 "Zone Other Equipment Total Heating Energy",
-                "Hourly",
+                "Hourly",  # as defined in idf
             ],
             [
                 "OUTPUT:VARIABLE",
                 "Block1:ApptX1W Ideal Loads Air",
                 "Zone Ideal Loads Supply Air Total Heating Energy",
-                "Hourly",
+                "Timestep",
             ],
             [
                 "OUTPUT:VARIABLE",
                 "Block1:ApptX1E Ideal Loads Air",
                 "Zone Ideal Loads Supply Air Total Heating Energy",
-                "Hourly",
+                "Timestep",
             ],
         ]
 
@@ -188,7 +188,7 @@ def test_heating_auxiliary(self):
                 "OUTPUT:VARIABLE",
                 "Block1:ApptX1W Ideal Loads Air",
                 "Zone Ideal Loads Supply Air Total Heating Energy",
-                "Hourly",
+                "Timestep",
             ],
         ]
         energyplus_results = read_eplus_res(RESOURCES_PATH / "test_res.csv")
@@ -250,11 +250,14 @@ def test_ahu(self):
                 "outputs": "SYSTEM",
             },
         )
-
-        assert results.sum().to_dict() == {
-            "TOTAL_SYSTEM_Energy_[J]": 634902466.3027192,
-            "VENTILATION_Energy_[J]": 634902466.3027192,
-        }
+        # results outputs : 30min
+        assert results.sum().to_dict() == approx(
+            {
+                "TOTAL_SYSTEM_Energy_[J]": 634902466.3027192 * 2,
+                "VENTILATION_Energy_[J]": 634902466.3027192 * 2,
+            },
+            rel=0.05,
+        )
 
     def test_dhw_ideal_external(self):
         building = Building(idf_path=RESOURCES_PATH / "test.idf")
@@ -270,10 +273,13 @@ def test_dhw_ideal_external(self):
             },
         )
 
-        assert results.sum().to_dict() == {
-            "DHW_Energy_[J]": 8430408987.330694,
-            "TOTAL_SYSTEM_Energy_[J]": 8430408987.330694,
-        }
+        assert results.sum().to_dict() == approx(
+            {
+                "DHW_Energy_[J]": 8430408987.330694,
+                "TOTAL_SYSTEM_Energy_[J]": 8430408987.330694,
+            },
+            rel=0.05,
+        )
 
     def test_artificial_lighting(self):
         building = Building(idf_path=RESOURCES_PATH / "test.idf")
@@ -337,10 +343,13 @@ def test_ahu_control(self):
             },
         )
 
-        assert results.sum().to_dict() == {
-            "TOTAL_SYSTEM_Energy_[J]": 5800244198.831992,
-            "VENTILATION_Energy_[J]": 5800244198.831992,
-        }
+        assert results.sum().to_dict() == approx(
+            {
+                "TOTAL_SYSTEM_Energy_[J]": 5800244198.831992*2,
+                "VENTILATION_Energy_[J]": 5800244198.831992*2,
+            },
+            rel=0.05,
+        )
 
     def test_other_equipments(self):
         tested_idf = IDF(RESOURCES_PATH / "test.idf")