Merge pull request #99 from RWTH-EBC/98-add-save-options-for-dymolaapi [PYPI-RELEASE]

feat: Add experiment setup output option

Author: FWuellhorst

.gitlab-ci.yml

@@ -12,6 +12,7 @@ variables:
   PYTHON_VERSION: "registry.git.rwth-aachen.de/ebc/ebc_all/gitlab_ci/templates:python_3.9"
   TEST_ENGINE: "unittest"
   GIT_REPO: "RWTH-EBC/ebcpy"
+  EXCLUDE_PYTHON: 37
 
 include:
   - project: 'EBC/EBC_all/gitlab_ci/templates'

CHANGELOG.md

@@ -112,4 +112,7 @@
   - Added bayesian-optimization to the possible optimizers #129
 - v0.5.1
   - Add save-logs option for SimulatioAPI #141
-  - Minor bug fixes #143
+  - Minor bug fixes #143
+- v0.5.2
+  - Add save options in Dymola #98
+  - Add feature to postprocess mat results within the simulate function to avoid memory errors in large studies

ebcpy/__init__.py

@@ -8,4 +8,4 @@
 from .optimization import Optimizer
 
 
-__version__ = '0.5.1'
+__version__ = '0.5.2'

ebcpy/simulationapi/__init__.py

@@ -393,7 +393,7 @@ def _remaining_time(self, t1):
             Start time after n_cpu simulations.
         """
         t_remaining = (time.time() - t1) / (self._n_sim_counter - self.n_cpu) * (
-                    self._n_sim_total - self._n_sim_counter)
+                self._n_sim_total - self._n_sim_counter)
         p_finished = self._n_sim_counter / self._n_sim_total * 100
         sys.stderr.write(f"\rFinished {np.round(p_finished, 1)} %. "
                          f"Approximately remaining time: {timedelta(seconds=int(t_remaining))} ")
@@ -414,6 +414,13 @@ def convert_bytes(size):
                 size = size / 1024.0
             return f'{str(np.round(size, 2))} {suffixes[suffix_idx]}'
 
+        if not isinstance(filepath, (Path, str)) or not os.path.exists(filepath):
+            self.logger.info(
+                "Can't check disk usage as you probably used postprocessing on simulation "
+                "results but did not return a file-path in the post-processing function"
+            )
+            return
+
         sim_file_size = os.stat(filepath).st_size
         sim_files_size = sim_file_size * self._n_sim_total
         self.logger.info(f"Simulations files need approximately {convert_bytes(sim_files_size)} of disk space")
@@ -444,7 +451,7 @@ def set_sim_setup(self, sim_setup):
         """
         Replaced in v0.1.7 by property function
         """
-        new_setup = self._sim_setup.dict()
+        new_setup = self._sim_setup.model_dump()
         new_setup.update(sim_setup)
         self._sim_setup = self._sim_setup_class(**new_setup)
 

ebcpy/simulationapi/dymola_api.py

@@ -14,7 +14,7 @@
 from contextlib import closing
 from typing import Union, List
 
-from pydantic import Field
+from pydantic import Field, BaseModel
 import pandas as pd
 
 from ebcpy import TimeSeriesData
@@ -42,6 +42,27 @@ class DymolaSimulationSetup(SimulationSetup):
                         "Radau", "Dopri45", "Dopri853", "Sdirk34hw"]
 
 
+class ExperimentSetupOutput(BaseModel):
+    """
+    Experiment setup output data model with
+    defaults equal to those in Dymola
+    """
+    states: bool = True
+    derivatives: bool = True
+    inputs: bool = True
+    outputs: bool = True
+    auxiliaries: bool = True
+    equidistant: bool = False
+    events: bool = True
+
+    class Config:
+        """
+        Pydantic internal model settings
+        """
+        # pylint: disable=too-few-public-methods
+        extra = "forbid"
+
+
 class DymolaAPI(SimulationAPI):
     """
     API to a Dymola instance.
@@ -63,6 +84,15 @@ class DymolaAPI(SimulationAPI):
     :keyword Boolean equidistant_output:
         If True (Default), Dymola stores variables in an
         equisdistant output and does not store variables at events.
+    :keyword dict[str,bool] variables_to_save:
+        A dictionary to select which variables are going
+        to be stored if the simulation creates .mat files.
+        Options (with the default being all True):
+            - states=True
+            - derivatives=True
+            - inputs=True
+            - outputs=True
+            - auxiliaries=False
     :keyword int n_restart:
         Number of iterations after which Dymola should restart.
         This is done to free memory. Default value -1. For values
@@ -144,6 +174,7 @@ class DymolaAPI(SimulationAPI):
         "modify_structural_parameters",
         "dymola_path",
         "equidistant_output",
+        "variables_to_save",
         "n_restart",
         "debug",
         "mos_script_pre",
@@ -170,6 +201,9 @@ def __init__(
         self.show_window = kwargs.pop("show_window", False)
         self.modify_structural_parameters = kwargs.pop("modify_structural_parameters", True)
         self.equidistant_output = kwargs.pop("equidistant_output", True)
+        _variables_to_save = kwargs.pop("variables_to_save", {})
+        self.experiment_setup_output = ExperimentSetupOutput(**_variables_to_save)
+
         self.mos_script_pre = kwargs.pop("mos_script_pre", None)
         self.mos_script_post = kwargs.pop("mos_script_post", None)
         self.dymola_version = kwargs.pop("dymola_version", None)
@@ -277,7 +311,8 @@ def __init__(
         if not self.license_is_available():
             warnings.warn("You have no licence to use Dymola. "
                           "Hence you can only simulate models with 8 or less equations.")
-
+        # Update experiment setup output
+        self.update_experiment_setup_output(self.experiment_setup_output)
         self.fully_initialized = True
         # Trigger on init.
         if model_name is not None:
@@ -325,6 +360,15 @@ def simulate(self,
             If inputs are given, you have to specify the file_name of the table
             in the instance of CombiTimeTable. In order for the inputs to
             work the value should be equal to the value of 'fileName' in Modelica.
+        :keyword callable postprocess_mat_result:
+            When choosing return_option savepath and no equidistant output, the mat files may take up
+            a lot of disk space while you are only interested in some variables or parts
+            of the simulation results. This features enables you to pass any function which
+            gets the mat-path as an input and returns some result you are interested in.
+            The function signature is `foo(mat_result_file, **kwargs_postprocessing) -> Any`.
+            Be sure to define the function in a global scope to allow multiprocessing.
+        :keyword dict kwargs_postprocessing:
+            Keyword arguments used in the function `postprocess_mat_result`.
         :keyword List[str] structural_parameters:
             A list containing all parameter names which are structural in Modelica.
             This means a modifier has to be created in order to change
@@ -374,6 +418,12 @@ def _single_simulation(self, kwargs):
         table_name = kwargs.pop("table_name", None)
         file_name = kwargs.pop("file_name", None)
         savepath = kwargs.pop("savepath", None)
+
+        def empty_postprocessing(mat_result, **_kwargs):
+            return mat_result
+
+        postprocess_mat_result = kwargs.pop("postprocess_mat_result", empty_postprocessing)
+        kwargs_postprocessing = kwargs.pop("kwargs_postprocessing", {})
         if kwargs:
             self.logger.error(
                 "You passed the following kwargs which "
@@ -388,9 +438,14 @@ def _single_simulation(self, kwargs):
                 # method used in the DymolaInterface should work.
                 self._setup_dymola_interface(dict(use_mp=True))
 
+            # Re-set the dymola experiment output if API is newly started
+            self.dymola.experimentSetupOutput(**self.experiment_setup_output.model_dump())
+
         # Handle eventlog
         if show_eventlog:
-            self.dymola.experimentSetupOutput(events=True)
+            if not self.experiment_setup_output.events:
+                raise ValueError("You can't log events and have an "
+                                 "equidistant output, set equidistant output=False")
             self.dymola.ExecuteCommand("Advanced.Debug.LogEvents = true")
             self.dymola.ExecuteCommand("Advanced.Debug.LogEventsInitialization = true")
 
@@ -582,23 +637,26 @@ def _single_simulation(self, kwargs):
             self.dymola.cd()
             # Get the value and convert it to a 100 % fitting str-path
             dymola_working_directory = str(Path(self.dymola.getLastErrorLog().replace("\n", "")))
+            mat_working_directory = os.path.join(dymola_working_directory, _save_name_dsres)
             if savepath is None or str(savepath) == dymola_working_directory:
-                return os.path.join(dymola_working_directory, _save_name_dsres)
-            os.makedirs(savepath, exist_ok=True)
-            for filename in [_save_name_dsres]:
+                mat_result_file = mat_working_directory
+            else:
+                mat_save_path = os.path.join(savepath, _save_name_dsres)
+                os.makedirs(savepath, exist_ok=True)
                 # Copying dslogs and dsfinals can lead to errors,
                 # as the names are not unique
                 # for filename in [_save_name_dsres, "dslog.txt", "dsfinal.txt"]:
                 # Delete existing files
                 try:
-                    os.remove(os.path.join(savepath, filename))
+                    os.remove(mat_save_path)
                 except OSError:
                     pass
                 # Move files
-                shutil.copy(os.path.join(dymola_working_directory, filename),
-                            os.path.join(savepath, filename))
-                os.remove(os.path.join(dymola_working_directory, filename))
-            return os.path.join(savepath, _save_name_dsres)
+                shutil.copy(mat_working_directory, mat_save_path)
+                os.remove(mat_working_directory)
+                mat_result_file = mat_save_path
+            result_file = postprocess_mat_result(mat_result_file, **kwargs_postprocessing)
+            return result_file
 
         data = res[1]  # Get data
         if return_option == "last_point":
@@ -832,17 +890,36 @@ def _setup_dymola_interface(self, kwargs: dict):
             if not res:
                 raise ImportError(dymola.getLastErrorLog())
         self.logger.info("Loaded modules")
-        if self.equidistant_output:
-            # Change the Simulation Output, to ensure all
-            # simulation results have the same array shape.
-            # Events can also cause errors in the shape.
-            dymola.experimentSetupOutput(equidistant=True,
-                                         events=False)
+
+        dymola.experimentSetupOutput(**self.experiment_setup_output.dict())
         if use_mp:
             DymolaAPI.dymola = dymola
             return None
         return dymola
 
+    def update_experiment_setup_output(self, experiment_setup_output: Union[ExperimentSetupOutput, dict]):
+        """
+        Function to update the ExperimentSetupOutput in Dymola for selection
+        of which variables are going to be saved. The options
+        `events` and `equidistant` are overridden if equidistant output is required.
+
+        :param (ExperimentSetupOutput, dict) experiment_setup_output:
+            An instance of ExperimentSetupOutput or a dict with valid keys for it.
+        """
+        if isinstance(experiment_setup_output, dict):
+            self.experiment_setup_output = ExperimentSetupOutput(**experiment_setup_output)
+        else:
+            self.experiment_setup_output = experiment_setup_output
+        if self.equidistant_output:
+            # Change the Simulation Output, to ensure all
+            # simulation results have the same array shape.
+            # Events can also cause errors in the shape.
+            self.experiment_setup_output.equidistant = True
+            self.experiment_setup_output.events = False
+        if self.dymola is None:
+            return
+        self.dymola.experimentSetupOutput(**self.experiment_setup_output.model_dump())
+
     def license_is_available(self, option: str = "Standard"):
         """Check if license is available"""
         if self.dymola is None:
@@ -1215,7 +1292,7 @@ def _check_dymola_instances(self):
             try:
                 if "Dymola" in proc.name():
                     counter += 1
-            except psutil.AccessDenied:
+            except (psutil.AccessDenied, psutil.NoSuchProcess):
                 continue
         if counter >= self._critical_number_instances:
             warnings.warn("There are currently %s Dymola-Instances "

ebcpy/utils/reproduction.py

@@ -9,10 +9,11 @@
 import platform
 import os
 import logging
-from typing import List, Union
+from typing import List, Tuple, Union
 import zipfile
 from datetime import datetime
 from dataclasses import dataclass
+from importlib.metadata import distributions
 
 logger = logging.getLogger(__name__)
 
@@ -322,44 +323,58 @@ def _get_general_information(title: str, log_message: str, current_time: str):
     return "\n".join(_content_lines)
 
 
+def get_installed_packages() -> List[dict]:
+    """
+    Returns a list of tuples containing (package_name, version, location)
+    for all installed Python packages.
+    """
+    packages = []
+    for dist in distributions():
+        packages.append(dict(
+            name=dist.metadata['Name'],
+            version=dist.version,
+            location=os.path.normpath(str(dist.locate_file('')))
+        ))
+    return packages
+
+
 def _get_python_package_information(search_on_pypi: bool):
     """
     Function to get the content of python packages installed
     as a requirement.txt format content.
     """
-    import pkg_resources
-    installed_packages = [pack for pack in pkg_resources.working_set]
+    installed_packages = get_installed_packages()
     diff_paths = []
     requirement_txt_content = []
     pip_version = ""
     for package in installed_packages:
         repo_info = get_git_information(
-            path=package.location,
-            name=package.key,
+            path=package["location"],
+            name=package["name"],
             zip_folder_path="python"
         )
         if repo_info is None:
             # Check if in python path:
-            if package.key == "pip":  # exclude pip in requirements and give info to _get_python_reproduction
-                pip_version = f"=={package.version}"
+            if package["name"] == "pip":  # exclude pip in requirements and give info to _get_python_reproduction
+                pip_version = f'=={package["version"]}'
             else:
                 requirement_txt_content.append(
-                    f"{package.key}=={package.version}"
+                    f'{package["name"]}=={package["version"]}'
                 )
             if search_on_pypi:
                 from pypisearch.search import Search
-                res = Search(package.key).result
+                res = Search(package["name"]).result
                 if not res:
                     raise ModuleNotFoundError(
                         "Package '%s' is neither a git "
                         "repo nor a package on pypi. "
                         "Won't be able to reproduce it!",
-                        package.key
+                        package["name"]
                     )
         else:
             cmt_sha = repo_info["commit"]
             requirement_txt_content.append(
-                f"git+{repo_info['url']}.git@{cmt_sha}#egg={package.key}"
+                f"git+{repo_info['url']}.git@{cmt_sha}#egg={package['name']}"
             )
             diff_paths.extend(repo_info["difference_files"])
     return "\n".join(requirement_txt_content), diff_paths, pip_version