Improved sample handling (#293)

* fix for appending samples from loaded models

* added unit tests

* use most recent version of refl1d

* update docs to include add_sample_from_orso

* temporarily revert to fwhm

* minor fixes after CR + ruff format

* back to pointwise

* proper conversion from stack to layers.

* added method

* temporarily switch off pointwise

* fixed SLD read.

* added handling for orso names (model and experiment)

* minor fix for names

* CR review fixes

* model reindexing issue fixed. Project rst added

* path for SLD-less orso files

* fix variances sent to bumps. Make FWHM default (temporarily)

* fixed test to correspond to new resolution

* added logs for changing minimizers

* reload constraints where necessary

* expose chi2 properly

* PR review #1

* Fix the zero variances issue

* allow 3.13

* added some tests

Author: rozyczko

.copier-answers.yml

@@ -2,7 +2,7 @@
 _commit: 8bdcedc
 _src_path: gh:/EasyScience/EasyProjectTemplate
 description: A reflectometry python package built on the EasyScience framework.
-max_python: '3.12'
+max_python: '3.13'
 min_python: '3.9'
 orgname: EasyScience
 packagename: easyreflectometry

.github/workflows/python-ci.yml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.11', '3.12']
+        python-version: ['3.11', '3.12', '3.13']
         os: [ubuntu-latest, macos-latest, windows-2022]
 
     runs-on: ${{ matrix.os }}

.github/workflows/python-package.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.11','3.12']
+        python-version: ['3.11','3.12','3.13']
     if: "!contains(github.event.head_commit.message, '[ci skip]')"
 
     steps:

CONTRIBUTING.rst

@@ -102,7 +102,7 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the list in README.md.
-3. The pull request should work for Python, 3.11 and 3.12, and for PyPy. Check
+3. The pull request should work for Python, 3.11, 3.12, and 3.13, and for PyPy. Check
    https://travis-ci.com/easyScience/EasyReflectometryLib/pull_requests
    and make sure that the tests pass for all supported Python versions.
 

pyproject.toml

@@ -23,10 +23,11 @@ classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Development Status :: 3 - Alpha"
 ]
 
-requires-python = ">=3.11,<3.13"
+requires-python = ">=3.11,<3.14"
 
 dependencies = [
     "easyscience @ git+https://github.com/easyscience/corelib.git@develop",
@@ -134,11 +135,12 @@ force-single-line = true
 legacy_tox_ini = """
 [tox]
 isolated_build = True
-envlist = py{3.11,3.12}
+envlist = py{3.11,3.12,3.13}
 [gh-actions]
 python =
         3.11: py311
         3.12: py312
+        3.13: py313
 [gh-actions:env]
 PLATFORM =
     ubuntu-latest: linux

src/easyreflectometry/fitting.py

@@ -31,6 +31,7 @@ def wrapped(*args, **kwargs):
         self._fit_func = [func_wrapper(m.interface.fit_func, m.unique_name) for m in args]
         self._models = args
         self.easy_science_multi_fitter = EasyScienceMultiFitter(args, self._fit_func)
+        self._fit_results: list[FitResults] | None = None
 
     def fit(self, data: sc.DataGroup, id: int = 0) -> sc.DataGroup:
         """
@@ -75,6 +76,7 @@ def fit(self, data: sc.DataGroup, id: int = 0) -> sc.DataGroup:
             dy.append(1 / np.sqrt(variances_masked))
 
         result = self.easy_science_multi_fitter.fit(x, y, weights=dy)
+        self._fit_results = result
         new_data = data.copy()
         for i, _ in enumerate(result):
             id = refl_nums[i]
@@ -99,7 +101,53 @@ def fit_single_data_set_1d(self, data: DataSet1D) -> FitResults:
         :param data: DataGroup to be fitted to and populated
         :param method: Optimisation method
         """
-        return self.easy_science_multi_fitter.fit(x=[data.x], y=[data.y], weights=[data.ye])[0]
+        x_vals = np.asarray(data.x)
+        y_vals = np.asarray(data.y)
+        variances = np.asarray(data.ye)
+
+        zero_variance_mask = variances == 0.0
+        num_zero_variance = int(np.sum(zero_variance_mask))
+
+        if num_zero_variance > 0:
+            warnings.warn(
+                f'Masked {num_zero_variance} data point(s) in single-dataset fit due to zero variance during fitting.',
+                UserWarning,
+            )
+
+        valid_mask = ~zero_variance_mask
+        if not np.any(valid_mask):
+            raise ValueError('Cannot fit single dataset: all points have zero variance.')
+
+        x_vals_masked = x_vals[valid_mask]
+        y_vals_masked = y_vals[valid_mask]
+        variances_masked = variances[valid_mask]
+
+        weights = 1.0 / np.sqrt(variances_masked)
+        result = self.easy_science_multi_fitter.fit(x=[x_vals_masked], y=[y_vals_masked], weights=[weights])[0]
+        self._fit_results = [result]
+        return result
+
+    @property
+    def chi2(self) -> float | None:
+        """Total chi-squared across all fitted datasets, or None if no fit has been performed."""
+        if self._fit_results is None:
+            return None
+        return sum(r.chi2 for r in self._fit_results)
+
+    @property
+    def reduced_chi(self) -> float | None:
+        """Reduced chi-squared from the most recent fit, or None if no fit has been performed."""
+        if self._fit_results is None:
+            return None
+        total_chi2 = sum(r.chi2 for r in self._fit_results)
+        total_points = sum(np.size(r.x) for r in self._fit_results)
+        n_params = self._fit_results[0].n_pars
+        total_dof = total_points - n_params
+
+        if total_dof <= 0:
+            return None
+
+        return total_chi2 / total_dof
 
     def switch_minimizer(self, minimizer: AvailableMinimizers) -> None:
         """

src/easyreflectometry/project.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import logging
 import os
 from pathlib import Path
 from typing import Dict
@@ -10,8 +11,8 @@
 import numpy as np
 from easyscience import global_object
 from easyscience.fitting import AvailableMinimizers
-from easyscience.fitting.fitter import DEFAULT_MINIMIZER
 from easyscience.variable import Parameter
+from easyscience.variable.parameter_dependency_resolver import resolve_all_parameter_dependencies
 from scipp import DataGroup
 
 from easyreflectometry.calculators import CalculatorFactory
@@ -20,23 +21,23 @@
 from easyreflectometry.data.measurement import extract_orso_title
 from easyreflectometry.data.measurement import load_data_from_orso_file
 from easyreflectometry.fitting import MultiFitter
-from easyreflectometry.model import LinearSpline
 from easyreflectometry.model import Model
 from easyreflectometry.model import ModelCollection
 from easyreflectometry.model import PercentageFwhm
-from easyreflectometry.model import Pointwise
 from easyreflectometry.sample import Layer
 from easyreflectometry.sample import Material
 from easyreflectometry.sample import MaterialCollection
 from easyreflectometry.sample import Multilayer
 from easyreflectometry.sample import Sample
 from easyreflectometry.sample.collections.base_collection import BaseCollection
 
+logger = logging.getLogger(__name__)
+
 Q_MIN = 0.001
 Q_MAX = 0.3
 Q_RESOLUTION = 500
 
-DEFAULT_MINIZER = AvailableMinimizers.LMFit_leastsq
+DEFAULT_MINIMIZER = AvailableMinimizers.LMFit_leastsq
 
 
 class Project:
@@ -48,6 +49,7 @@ def __init__(self):
         self._calculator = CalculatorFactory()
         self._experiments: Dict[DataGroup] = {}
         self._fitter: MultiFitter = None
+        self._minimizer_selection: AvailableMinimizers = DEFAULT_MINIMIZER
         self._colors: list[str] = None
         self._report = None
         self._q_min: float = None
@@ -207,9 +209,8 @@ def models(self, models: ModelCollection) -> None:
     def fitter(self) -> MultiFitter:
         if len(self._models):
             if (self._fitter is None) or (self._fitter_model_index != self._current_model_index):
-                minimizer = self.minimizer
                 self._fitter = MultiFitter(self._models[self._current_model_index])
-                self.minimizer = minimizer
+                self._fitter.easy_science_multi_fitter.switch_minimizer(self._minimizer_selection)
                 self._fitter_model_index = self._current_model_index
         return self._fitter
 
@@ -225,10 +226,14 @@ def calculator(self, calculator: str) -> None:
     def minimizer(self) -> AvailableMinimizers:
         if self._fitter is not None:
             return self._fitter.easy_science_multi_fitter.minimizer.enum
-        return DEFAULT_MINIMIZER
+        return self._minimizer_selection
 
     @minimizer.setter
     def minimizer(self, minimizer: AvailableMinimizers) -> None:
+        old_name = getattr(self._minimizer_selection, 'name', str(self._minimizer_selection))
+        new_name = getattr(minimizer, 'name', str(minimizer))
+        logger.info('Minimizer changed from %s to %s (fitter active: %s)', old_name, new_name, self._fitter is not None)
+        self._minimizer_selection = minimizer
         if self._fitter is not None:
             self._fitter.easy_science_multi_fitter.switch_minimizer(minimizer)
 
@@ -386,21 +391,10 @@ def _apply_resolution_function(
     ) -> None:
         """Set the resolution function on *model* based on variance data in *experiment*.
 
-        Prefers Pointwise when q-resolution (xe) data is present, otherwise falls
-        back to LinearSpline when reflectivity error (ye) data is present.
-
         :param experiment: The experiment whose variance data drives the choice.
         :param model: The model whose resolution function is set.
         """
-        if sum(experiment.xe) != 0:
-            resolution_function = Pointwise(q_data_points=[experiment.x, experiment.y, experiment.xe])
-            model.resolution_function = resolution_function
-        elif sum(experiment.ye) != 0:
-            resolution_function = LinearSpline(
-                q_data_points=experiment.x,
-                fwhm_values=np.sqrt(experiment.ye),
-            )
-            model.resolution_function = resolution_function
+        model.resolution_function = PercentageFwhm(5.0)
 
     def load_new_experiment(self, path: Union[Path, str]) -> None:
         new_experiment = load_as_dataset(str(path))
@@ -603,6 +597,8 @@ def as_dict(self, include_materials_not_in_model=False):
             self._as_dict_add_experiments(project_dict)
         if self.fitter is not None:
             project_dict['fitter_minimizer'] = self.fitter.easy_science_multi_fitter.minimizer.name
+        elif self._minimizer_selection is not None:
+            project_dict['fitter_minimizer'] = self._minimizer_selection.name
         if self._calculator is not None:
             project_dict['calculator'] = self._calculator.current_interface_name
         if self._colors is not None:
@@ -641,14 +637,17 @@ def from_dict(self, project_dict: dict):
         if 'materials_not_in_model' in keys:
             self._materials.extend(MaterialCollection.from_dict(project_dict['materials_not_in_model']))
         if 'fitter_minimizer' in keys:
-            self.fitter.easy_science_multi_fitter.switch_minimizer(AvailableMinimizers[project_dict['fitter_minimizer']])
+            self.minimizer = AvailableMinimizers[project_dict['fitter_minimizer']]
         else:
             self._fitter = None
         if 'experiments' in keys:
             self._experiments = self._from_dict_extract_experiments(project_dict)
         else:
             self._experiments = {}
 
+        # Resolve any pending parameter dependencies (constraints) after all objects are loaded
+        resolve_all_parameter_dependencies(self)
+
     def _from_dict_extract_experiments(self, project_dict: dict) -> Dict[int, DataSet1D]:
         experiments = {}
         for key in project_dict['experiments'].keys():

tests/summary/test_summary.py

@@ -133,7 +133,7 @@ def test_experiments_section(self, project: Project) -> None:
         assert 'No. of data points' in html
         assert '408' in html
         assert 'Resolution function' in html
-        assert 'Pointwise' in html
+        assert 'PercentageFwhm' in html
 
     def test_experiments_section_percentage_fhwm(self, project: Project) -> None:
         # When